Refactored orchestrator for staged file handling, added structured prompt support, adjusted Feishu file handling
This commit is contained in:
whlaoding
3
.vscode/launch.json
vendored
3
.vscode/launch.json
vendored
@@ -6,6 +6,7 @@
|
||||
"type": "go",
|
||||
"request": "launch",
|
||||
"mode": "auto",
|
||||
"preLaunchTask": "Kill Stale LaodingBot Debug Processes",
|
||||
"program": "${workspaceFolder}/cmd/bot",
|
||||
"cwd": "${workspaceFolder}",
|
||||
"envFile": "${workspaceFolder}/configs/env"
|
||||
@@ -15,6 +16,7 @@
|
||||
"type": "go",
|
||||
"request": "launch",
|
||||
"mode": "auto",
|
||||
"preLaunchTask": "Kill Stale LaodingBot Debug Processes",
|
||||
"program": "${workspaceFolder}/cmd/bot",
|
||||
"cwd": "${workspaceFolder}",
|
||||
"envFile": "${workspaceFolder}/configs/env",
|
||||
@@ -27,6 +29,7 @@
|
||||
"type": "go",
|
||||
"request": "launch",
|
||||
"mode": "auto",
|
||||
"preLaunchTask": "Kill Stale LaodingBot Debug Processes",
|
||||
"program": "${workspaceFolder}/cmd/bot",
|
||||
"cwd": "${workspaceFolder}",
|
||||
"envFile": "${workspaceFolder}/configs/env",
|
||||
|
||||
11
.vscode/tasks.json
vendored
Normal file
11
.vscode/tasks.json
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"version": "2.0.0",
|
||||
"tasks": [
|
||||
{
|
||||
"label": "Kill Stale LaodingBot Debug Processes",
|
||||
"type": "shell",
|
||||
"command": "pkill -f '/LaodingBot/cmd/bot/__debug_bin|dlv.*LaodingBot/cmd/bot|dlv dap' || true",
|
||||
"problemMatcher": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -5,7 +5,7 @@ Go-based personal Telegram Agent with:
|
||||
- Telegram polling transport
|
||||
- OpenAI-compatible LLM client
|
||||
- SQLite conversation memory + simple compression
|
||||
- Tool registry with built-in `file` and `shell` tools
|
||||
- Tool registry with built-in `file`, `shell`, and `git` tools
|
||||
- Default-deny security policy via allowlists
|
||||
- Soul markdown loading for bot personality
|
||||
- Skills markdown loading for capability context
|
||||
@@ -50,7 +50,7 @@ go run ./cmd/bot
|
||||
1. Receive message and load recent memory context
|
||||
2. Match relevant skill(s) from `skills/`
|
||||
3. If no skill matched, respond via direct LLM
|
||||
4. If skill matched, run ReAct and call tools (`shell` / `file`) only when needed
|
||||
4. If skill matched, run ReAct and call tools (`shell` / `file` / `git`) only when needed
|
||||
5. Return final answer
|
||||
- No `/tool ...` command is required for normal use.
|
||||
|
||||
@@ -69,6 +69,7 @@ go run ./cmd/bot
|
||||
|
||||
- `shell` only allows commands listed in `ALLOWED_COMMANDS`.
|
||||
- `file` only allows paths inside `ALLOWED_DIRS`.
|
||||
- `git` only allows common git subcommands and runs inside `WORK_DIR`.
|
||||
- Working directory for shell is limited by `WORK_DIR`.
|
||||
|
||||
## Next Iteration
|
||||
|
||||
@@ -177,6 +177,24 @@ func runMessageChannel(ctx context.Context, cfg config.Config, engine *agent.Orc
|
||||
}
|
||||
lg.Infof("starting feishu transport")
|
||||
return fs.Run(ctx, func(ctx context.Context, msg feishu.IncomingMessage) (string, error) {
|
||||
if msg.MsgType == "file" && len(msg.FileBytes) > 0 {
|
||||
content := msg.FileBytes
|
||||
if msg.FilePath != "" {
|
||||
if b, err := os.ReadFile(msg.FilePath); err == nil && len(b) > 0 {
|
||||
content = b
|
||||
} else if lg != nil {
|
||||
lg.Warnf("read local file failed path=%s err=%v; fallback to in-memory bytes", msg.FilePath, err)
|
||||
}
|
||||
}
|
||||
files := []llm.InputFile{{
|
||||
FileName: msg.FileName,
|
||||
MimeType: msg.FileMime,
|
||||
Content: content,
|
||||
}}
|
||||
// Feishu file event and user question are split into separate messages.
|
||||
// Use empty text so file IDs are cached and consumed by the next text query.
|
||||
return engine.HandleMessageWithFiles(ctx, msg.ChatID, msg.UserID, "", files)
|
||||
}
|
||||
return engine.HandleMessage(ctx, msg.ChatID, msg.UserID, msg.Text)
|
||||
})
|
||||
default:
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
2455
files/EIP_RequirementsSpecificationGLA__V2-5.txt
Normal file
2455
files/EIP_RequirementsSpecificationGLA__V2-5.txt
Normal file
File diff suppressed because it is too large
Load Diff
250
files/core_requirements_part1.txt
Normal file
250
files/core_requirements_part1.txt
Normal file
@@ -0,0 +1,250 @@
|
||||
|
||||
|
||||
Sub-Goal 5: User’s experience is
|
||||
enhanced by the provision of Use Cases: 3 Requirements: 8
|
||||
value-added services
|
||||
|
||||
|
||||
|
||||
Figure 2 Structure of this document.
|
||||
|
||||
|
||||
|
||||
|
||||
1.6 Product Scope and Perspective
|
||||
The EIP Project’s urban platform is an open common architecture which serves for city data
|
||||
collection, management and distribution. An urban platform is intended to support the widespread
|
||||
exploitation of city data by humans and machines in the urban environment. Figure 3 illustrates a
|
||||
holistic high level overview of the urban platform the EIP project intends to deliver.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 6
|
||||
|
||||
|
||||
|
||||
The reference architecture for urban platforms should:
|
||||
|
||||
• Cater for interoperability between urban infrastructures
|
||||
• Enable replicability of the solutions/platforms city to city
|
||||
• Scale without technical constraints and excessive cost increase
|
||||
• Provide open APIs and SDKs
|
||||
• Enable Real Time capabilities
|
||||
• Support implementation of functional and technical capabilities
|
||||
|
||||
|
||||
|
||||
|
||||
Figure 3. High level overview of the urban platform (currently approved EC DG CNECT).
|
||||
|
||||
|
||||
The current urban platform market is nascent. Many software vendors offer such a platform, though
|
||||
many requirements and expectations of the stakeholders of city data are not (fully) addressed. As
|
||||
a result, current urban platforms are often more costly to design and maintain, less reusable and
|
||||
often not interoperable platform-to-platform, and susceptible to information fragmentation and
|
||||
overload.
|
||||
|
||||
The urban platform which the EIP initiative intends to design takes a step beyond the platforms
|
||||
currently on the market by ensuring the requirements are fully founded on a co-created and
|
||||
common set of representative city needs, from which it solicits suitable industry input, and an open
|
||||
and managed collaboration between industry, cities and communities, and others, in order to take
|
||||
into account their needs and concerns. To do this, it is necessary to take a technology agnostic
|
||||
approach to design an open and common reference architecture for urban platforms. This platform
|
||||
must ensure data is collected and sustained in accordance with well-stablished standards,
|
||||
managed in a robust manner so that it can handle high level supply and demand of data, and
|
||||
distributed across different value chains, systems and stakeholders. The ability to handle high level
|
||||
of city data supply and demand while being user secure and accessible enough for city-wide
|
||||
exploitation of data is one of many keys to the success of urban platforms. This is central to the
|
||||
design and implementation of urban platforms.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 7
|
||||
|
||||
|
||||
1.7 The Urban Platform Development Stack
|
||||
The requirements specification of this document is based on the Urban Platform Development
|
||||
Stack illustrated in Figure 4. The stack is composed by five domains (represented as layers in the
|
||||
stack) necessary to fully implement an urban platform software suite as shown in Table 1. Each
|
||||
domain comprehends a set of requirements necessary to the design of a common and open urban
|
||||
data service platform. The elicited requirements are used to define a technical architecture which is
|
||||
simple enough to be comprehensible at least at a high level of abstraction. The platform should be
|
||||
conceptually decomposable into its major subsystems, the platform’s functionality reused by many
|
||||
services and external applications should be identifiable, and interactions between the platform
|
||||
and services, data providers and data consumers should be well defined and explicit.
|
||||
|
||||
The first layer of the stack “Societal needs” concerns to outcomes we strive for within a portfolio of
|
||||
city service domains. An urban platform should recognise societal needs and wants as the starting
|
||||
point for city data service offering. Ultimately, an urban platform aims to provide tailor made and
|
||||
compelling engaging services for the users. The Services and Business models layers concerns
|
||||
with delivering data services which carefully targets the needs and expectations of the different
|
||||
users of the urban platform, and explore use cases and commercial models where data is used to
|
||||
deliver different forms of value. The city data layer concerns with the mechanisms necessary to
|
||||
transform urban platforms into a foundation for widespread exploitation of data, including handling
|
||||
data architectural features, data usability, semantics and quality aspects. The urban platform layer
|
||||
concerns to the technology foundation to configure, share, and interpret exponentially increasing
|
||||
volumes city data and services. Finally, the Infrastructure layer concerns with the base level
|
||||
connectivity that supports the platform to be scalable and reliable in the long run.
|
||||
|
||||
|
||||
|
||||
STACK OUTPUT
|
||||
|
||||
Requirements to deliver new digital services that will address the
|
||||
Societal Needs societal needs of cities in a positive manner that relates to political
|
||||
narratives.
|
||||
|
||||
|
||||
Services & Requirements to new profitable business models and the development
|
||||
Business Models of an increase range of new and engaging services in the smart cities.
|
||||
|
||||
|
||||
|
||||
City Data Requirements to provide all city data stakeholders ready access and
|
||||
delivery of all city data that unpins the decision making process in smart
|
||||
cities.
|
||||
|
||||
|
||||
Urban Platform Requirements to put in place applications together to build a foundation
|
||||
for the widespread exploitation of data.
|
||||
|
||||
|
||||
Infrastructure Requirements to deliver the backbone infrastructure that will be used to
|
||||
capture the opportunities of digital technology and data to enable
|
||||
transformation.
|
||||
|
||||
|
||||
Figure 4. Urban Platform Development Stack.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 8
|
||||
|
||||
|
||||
Table 1. Urban Platform Development Stack
|
||||
Layer Rationale
|
||||
Societal Needs - Accessible services and data necessary to solve social problems
|
||||
and drive innovation;
|
||||
- Parameters that influence user’s experience while interacting with
|
||||
services (e.g. usability, feeling of security and trust);
|
||||
Services and Business - Tailor-made data services which careful targets the needs of users
|
||||
Models and businesses;
|
||||
- New potential and cost-effective beneficial services that could be
|
||||
rolled out across cities of different sizes;
|
||||
- Use cases where data is used to deliver different forms of value.
|
||||
City Data - Data architectural features (e.g. volume, variety, temporal factors
|
||||
and sensitivity);
|
||||
- Data licensing, policies and regulations to exploit data to full effect;
|
||||
- Minimum metadata requirements;
|
||||
- Data usability and reusability aspects of humans and machines.
|
||||
Urban Platform - Holistic and interoperable solutions;
|
||||
- Integrated approaches which ensures that services fit together and
|
||||
that synergies can be exploited;
|
||||
- Data management mechanisms to ensure data integrity and
|
||||
compliance with data protection regulations
|
||||
- Extension capabilities to accommodate additional functionality at
|
||||
later stage at a fair and transparent cost.
|
||||
|
||||
|
||||
|
||||
1.8 User Classes, Characteristics, and User Access
|
||||
The users of the Urban Platform include end-users, such as the general Public, public and private
|
||||
organisations; data providers; service providers; and the platform providers who will be
|
||||
working with the providers of city data and services, and managing the content, defining policies
|
||||
and regulations of the platform. A crucial feature of an urban platform is the provision of the
|
||||
various access levels required by the different types of users. Particular uses need different
|
||||
access levels to some data than the general public. Data publishers will require access to the
|
||||
Urban Platform in order to ingest, administer, manage, preserve and access their resources. This
|
||||
will require multiple levels of access to city data and its respective metadata. Table 2 provides a
|
||||
description of each class of users.
|
||||
|
||||
|
||||
Table 2. Actors
|
||||
User Class Rationale
|
||||
Platform - Maintains the ecosystem of data, services and users;
|
||||
Provider - Defines standards, licenses and regulations and provides terms and conditions
|
||||
for platform usage and the commercial exploitation of data and services;
|
||||
- Decides who are allowed to join the value network of data and services
|
||||
providers;
|
||||
City Data - Publishes open and proprietary data into the platform;
|
||||
Publisher - Manages and maintain resources in the platform accordingly to terms and
|
||||
conditions.
|
||||
Data - Deploys open and commercial data services into the platform (e.g. data
|
||||
Services visualisation, data cleansing, data integration tools);
|
||||
Provider - Manages and maintain resources in the platform accordingly to terms and
|
||||
conditions.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 9
|
||||
|
||||
|
||||
City Data - Consumes open and proprietary data provided in the platform;
|
||||
Consumer - Uses open and commercial data services provided in the platform;
|
||||
- Provides feedback on data and services provision;
|
||||
|
||||
|
||||
1.8.1 End-User Access
|
||||
|
||||
City data consumers will need to access and use the city data residing in the Urban Platform. End-
|
||||
users will be able to search metadata and full text within datasets (when available), and obtain city
|
||||
data in open formats readily available to both humans and machines such as CSV, XML, JSON.
|
||||
Some end-users may require different access rights to city data. The 2 major end-user groups that
|
||||
have been identified are:
|
||||
|
||||
• Open data users, including both national and international users (humans and machines).
|
||||
Open access to some city data may be restricted by licensing terms (e.g. commercial data),
|
||||
embargo periods, copyright, etc.
|
||||
|
||||
• Private data users, which need to use the Urban Platform to obtain commercial city data. Data
|
||||
access is available via data subscriptions or when purchase requirements and licenses are
|
||||
waived by the data provider.
|
||||
|
||||
|
||||
1.8.2 City Data Publisher Access
|
||||
|
||||
A broad data provider level access is needed for stakeholders (humans and machines) working
|
||||
with the urban platform and their respective data in it. Basically, data publishers will carry out the
|
||||
following activities:
|
||||
• Data publication access, available to publishers adding new data and metadata, checking the
|
||||
quality of datasets, manipulating data, performing format conversions, defining data-access
|
||||
level, tariff for consumption when applicable, and licences.
|
||||
|
||||
• Data maintenance access, for publishers reviewing or editing appropriate data and metadata in
|
||||
the urban platform. Data publishers can view data and add to or edit metadata without
|
||||
changing the data itself. They should be provided with access to feedback from users to
|
||||
investigate problem in their resources (e.g. missing data, inconsistent metadata), and statistical
|
||||
information about how their resources are used by users.
|
||||
|
||||
|
||||
1.8.3 Data Services Provider Access
|
||||
|
||||
This is the second most restrictive access level providing rights to deploy services in the platform.
|
||||
Basically, data service providers will carry out the following activities:
|
||||
|
||||
• Data services deployment access, available to service providers adding new mechanisms or
|
||||
integrating new applications, testing and validating integration, defining data-access level and
|
||||
tariff for service usage.
|
||||
|
||||
• Data services maintenance access, for services providers reviewing, extending or editing
|
||||
applications in the urban platform. Data services providers can view their services deployed
|
||||
and add to or edit access level and tariff without having to deploy the services again. They
|
||||
should be provided with access to feedback from users to investigate problem in their services
|
||||
(e.g. bugs, scalability issues), and statistical information about how their services are used by
|
||||
users.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 10
|
||||
|
||||
|
||||
1.8.4 Platform Provider Access
|
||||
|
||||
This is the most restrictive access level providing ultimate rights to the system and is required for
|
||||
its management, development, and assigning appropriate rights to data and services providers.
|
||||
Policies and regulations, license agreements are also defined by the provider of the urban
|
||||
platform. Platform providers should be provided with the means to follow up on civic engagement
|
||||
(e.g. feedback, request for city data) and on the provision of city data and services.
|
||||
|
||||
|
||||
1.9 User Documentation
|
||||
• City Data Consumers: Provide license terms and conditions associated to consuming
|
||||
data and services provided in the platform, documentation of APIs and guide to
|
||||
discover city data in the platform.
|
||||
• City Data Providers: Provide data publication documentation describing the minimum
|
||||
metadata requirements, formats accepted, step-by-step guide to publish accurate city
|
||||
data.
|
||||
• Data Service Providers: Disclosure technical and architecture blueprint details in
|
||||
order share and outsource expertise, and partnerships, and integrate supporting
|
||||
partners’ solutions into the platform itself.
|
||||
|
||||
|
||||
1.10 Design and Implementation Constraints
|
||||
1.10.1 Design Constraints
|
||||
|
||||
• Lack of standards agreement for metadata representation.
|
||||
600
files/core_requirements_part2.txt
Normal file
600
files/core_requirements_part2.txt
Normal file
@@ -0,0 +1,600 @@
|
||||
• City data found in existing data catalogues may require special consideration concerning the
|
||||
type of formats and datasets that must be stored within the platform.
|
||||
• Requirements mismatch due to increased number of stakeholders involved in the design
|
||||
|
||||
|
||||
1.10.2 Implementation Constraints
|
||||
|
||||
• Evaluation and testing of software options is expected to occur prior to selection and
|
||||
implementation of a production urban platform.
|
||||
• Budget costs are unknown until evaluation of software options is completed.
|
||||
|
||||
|
||||
1.11 Assumptions, Alignment with other Action Clusters and Policies
|
||||
1.11.1 Assumptions
|
||||
|
||||
The assumptions in Table 3 have been identified by the Demand Side Engagement Stream as
|
||||
relevant to this Requirements Specification.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 11
|
||||
|
||||
|
||||
Table 3. Assumptions
|
||||
|
||||
# ASSUMPTION
|
||||
|
||||
1 The providers of city data and services will be responsible to maintain their resources in the platform.
|
||||
|
||||
2 All city data must meet the minimum metadata requirements and use the standards adopted by the platform.
|
||||
|
||||
3 The platform shall consider open Source as an optional commercial model, with open standards as a principle
|
||||
|
||||
4 The system design and architecture should minimize fragmentation of city data in the urban platform.
|
||||
|
||||
5 To the extent possible, automation should be used for the extraction of descriptive and technical metadata.
|
||||
|
||||
6 The platform must be designed in a way it accommodates additional functionality at later stage at a fair and transparent cost.
|
||||
|
||||
The platform must be a modular based architecture which relies on stable and well-defined open interfaces to ensure
|
||||
7
|
||||
interoperability between the platform, services and the applications provided by service providers.
|
||||
|
||||
The platform will offer open and well-documented API’s and clear service descriptions and contracts that is offered for reuse by
|
||||
8 another party to foster open innovation in the city, which means that developers and interested individuals openly utilize the
|
||||
resources provided.
|
||||
|
||||
9 Adopt open and published European and International standards where possible.
|
||||
|
||||
The platform must be flexible enough to accommodate different local, National and International data protection, licensing and
|
||||
10
|
||||
commercialization regulations.
|
||||
|
||||
11 Platform providers will monitor emerging technologies in order to maintain and improve the architecture.
|
||||
|
||||
|
||||
12 Platform providers will monitor emerging information standards, including metadata standards and data interface standards.
|
||||
|
||||
|
||||
13 Platform providers will monitor new commercial models for city data exploitation
|
||||
|
||||
|
||||
|
||||
|
||||
1.11.2 Alignment with Citizen’s Focus Action Cluster
|
||||
This specification document is aligned with the principles defined in the Citizen Focus5 Action
|
||||
Cluster of the European Innovation Partnership on Smart Cities and Communities. Citizen Focus’ is
|
||||
about “working together with citizens to realize public interests at the intersection of ICT, mobility
|
||||
and energy in an urban environment”. We recognize citizens as owners of and participants in
|
||||
the creation and delivery of city data and digital services, and we specify requirements to deliver
|
||||
new digital services that will address the societal needs of cities in a positive manner that relates to
|
||||
political narratives. Societal needs and wants are considered the starting point for city data service
|
||||
offering by the urban platform. The requirements in this document were elicited considering
|
||||
|
||||
• Human behaviour and needs as important as technology;
|
||||
• The services and data that solve social problems and drive innovation;
|
||||
• The mechanisms that make data and services more accessible to users;
|
||||
• The factors that influences user’s experience while interacting with services provided (e.g.
|
||||
usability, feeling of security and trust);
|
||||
|
||||
|
||||
5
|
||||
https://eu-smartcities.eu/node/1333
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 12
|
||||
|
||||
|
||||
1.11.3 Policies to be developed
|
||||
|
||||
The following policies have been identified by the Demand Side Engagement Stream as relevant to
|
||||
this Requirements Specification.
|
||||
|
||||
• Data formatting and Metadata Schemes: Urban platforms will require more expansive, robust
|
||||
and useful data encoding and conversion that what is available in existing data catalogues.
|
||||
Data preservation policies should be developed to allow data to be stored in formats that can
|
||||
be migrated, associated with metadata and ontologies to become both humans and machines
|
||||
readable and understandable, ongoing monitoring for data obsolescence, and migrating data to
|
||||
systems environments as needed to ensure their continued availability. Current Metadata
|
||||
schemes (e.g. open data, sensory data ontologies) should be reviewed to see if it meets
|
||||
current needs for city data management. It is possible that the needs of the urban platform
|
||||
designed here may require new or additional schemas.
|
||||
|
||||
• Data commercialisation: The commercial exploitation of city data and their funding models
|
||||
are unexplored concepts that we are committed to address. There is an urgent need to define
|
||||
license agreements and fair commercial and subscription models to allow interoperable open
|
||||
and proprietary data to co-exist in the platform.
|
||||
|
||||
• Data publication and services deployment: Policy development will be needed regarding
|
||||
ingesting data and deploying data services into the platform, including which users/machines
|
||||
will be authorized to submit data for publication, the minimum requirements for data submitted
|
||||
by open and proprietary data publishers, and the removal of resources and services from the
|
||||
platform.
|
||||
|
||||
|
||||
2. Urban Platform Value Proposition, Use Cases and
|
||||
Functional Requirements
|
||||
2.1 From Value Proposition to Platform Specifications
|
||||
This document uses goal-oriented modelling for eliciting, elaborating, structuring, specifying,
|
||||
documenting, and modifying requirements. Goals represent the objectives which the urban
|
||||
platform should achieve through cooperation of actors in the intended system and in the
|
||||
environment. They capture, at different levels of abstraction, the various objectives the urban
|
||||
platform under design should achieve. Through goals modelling we consider how the value
|
||||
proposition and intended solutions connects across the stack, how the urban platform meets city
|
||||
goals, why the system and its functionality are needed, and how the stakeholders’ interests may be
|
||||
addressed.
|
||||
|
||||
In our specification, we present the overall goal (the value proposition) that the urban platform
|
||||
should aim to achieve in order to be considered as a viable final product, and a set of sub-goals
|
||||
(intended solutions) it should maintain in the long run so that the overall goal can be unceasingly
|
||||
achieved. By using this approach, the low-level technical requirements can be traced back to high-
|
||||
level strategic objectives of the urban platform. The formal notations used in this document are:
|
||||
Achieve [“Name of Overall Goal”] and Maintain [“Name of Sub-Goal”]. The requirements of the
|
||||
Urban Platform are noted for each of the sub-goals, and are presented as a series of statements
|
||||
regarding the capabilities needed in to achieve the overall goal of the Urban Platform.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 13
|
||||
|
||||
|
||||
2.2 Overall Goal: “City data is exploited to its full potential”
|
||||
An urban platform is an organization of people and systems, which has accepted the responsibility
|
||||
to preserve city data and make it available for all the stakeholders of smart cities. Ultimately, an
|
||||
urban platform is a foundation for the full exploitation of city data. Hence, the major goal an Urban
|
||||
Platform must achieve is “city data is exploited to its full potential”. To achieve this high level goal,
|
||||
the urban platform must maintain in the long run the five sub-goals illustrated in Figure 3.
|
||||
|
||||
Each one of the defined sub-goals co-enables the achievement of the specified high-level (overall)
|
||||
goal of the urban platform. The sub-goals include the ingestion of city data, metadata generation,
|
||||
data management, data storage, access, preservation, and administration, provision of engaging
|
||||
services in the smart cities. These sub-goals are discussed in details in the following sections.
|
||||
|
||||
|
||||
|
||||
Achieve [City data is exploited How we achieve the platform overall goal?
|
||||
Why do we need the sub-goals?
|
||||
to its full effect]
|
||||
Goal
|
||||
|
||||
|
||||
|
||||
|
||||
Maintain [User s experience is
|
||||
Maintain [City data is provided
|
||||
enhanced by the provision of
|
||||
in a harmonised way]
|
||||
S ub-Goal 1 S ub-Goal 5 value-added services]
|
||||
co-ena bles
|
||||
|
||||
|
||||
|
||||
|
||||
Maintain [Resources are managed in Maintain [City data is offered
|
||||
a safe and intelligent manner] in an accessible manner]
|
||||
S ub-Goal 2 S ub-Goal 4
|
||||
|
||||
|
||||
|
||||
|
||||
Maintain [City data is orchestrated
|
||||
in a market place]
|
||||
S ub-Goal 3
|
||||
|
||||
|
||||
|
||||
|
||||
Figure 3. Platform High-Level Goal and its respective sub-goals.
|
||||
|
||||
|
||||
|
||||
2.2.1 Urban Platform Boundary
|
||||
|
||||
The use case diagram illustrated in Figure 4 identifies the boundaries between the actors (either
|
||||
automated or human) and the urban platform. We have arrived at the urban platform boundary by
|
||||
inspecting each business use case and determining, in conjunction with the stakeholders needs,
|
||||
which part of the business use case should be implemented and what part should be done by an
|
||||
outsourced product (e.g. Billing System) using the framework 4.
|
||||
|
||||
This task is technology agnostic and takes into account the abilities of the users/actors, the
|
||||
constraints, the goals of the urban platform. Table 4 maps out the use cases with their respective
|
||||
sub-goals and actors.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 14
|
||||
|
||||
|
||||
|
||||
|
||||
Manage Services
|
||||
|
||||
S ub-Goal 5
|
||||
|
||||
|
||||
Data Service
|
||||
Publish City Data
|
||||
Provider
|
||||
S ub-Goal 1
|
||||
|
||||
|
||||
<<include>> Deploy Data Services
|
||||
rules
|
||||
S ub-Goal 5
|
||||
|
||||
Authenticate in the
|
||||
Manage Resources <<include>>
|
||||
Platform Platform
|
||||
S ub-Goal 1 Provider Database
|
||||
<<include>> Manage Infrastructure System
|
||||
S ub-Goal 2
|
||||
|
||||
Utlise Data Services
|
||||
City Data <<include>>
|
||||
Publisher S ub-Goal 5
|
||||
|
||||
Store City Data Management
|
||||
<<include>>
|
||||
S ub-Goal 2
|
||||
Systems Services
|
||||
System
|
||||
|
||||
<<include>>
|
||||
City Data
|
||||
Consumer
|
||||
QoS Monitoring
|
||||
Transmit Data
|
||||
System
|
||||
Register in the Platform S ub-Goal 2
|
||||
|
||||
|
||||
|
||||
|
||||
<<include>>
|
||||
rules
|
||||
|
||||
<<extend>>
|
||||
rules
|
||||
Platform
|
||||
Provider
|
||||
rules
|
||||
|
||||
Commercialise Data
|
||||
Consume City Data
|
||||
Services City Data
|
||||
S ub-Goal 3 S ub-Goal 4
|
||||
Consumer
|
||||
<<extend>>
|
||||
|
||||
|
||||
Commercialise
|
||||
City Data Discover City Data
|
||||
S ub-Goal 3
|
||||
S ub-Goal 4
|
||||
|
||||
Billing
|
||||
Management
|
||||
System
|
||||
|
||||
|
||||
|
||||
Authenticate in the Register in the
|
||||
Platform Platform
|
||||
|
||||
|
||||
|
||||
|
||||
Authenticate Authenticate
|
||||
Register Consumers Register Publishers
|
||||
Consumers Publishers
|
||||
|
||||
|
||||
|
||||
|
||||
Figure 4. Simplistic overview of the use cases identified in the early stages of the platform design.
|
||||
|
||||
|
||||
|
||||
|
||||
Table 4. Use Cases Mapping with Sub-Goals and Actors
|
||||
Sub-Goals Use Cases ID Specialised Use Cases Actors
|
||||
HIGH-LEVEL GOAL: City data is exploited to its
|
||||
|
||||
|
||||
|
||||
|
||||
Publish City Data UC1 User publishes city data via data API’s
|
||||
City Data Publisher
|
||||
1. City data is Register as a publisher
|
||||
UC2 User manually uploads datasets
|
||||
provided in a in the Platform
|
||||
harmonised way
|
||||
full effect
|
||||
|
||||
|
||||
|
||||
|
||||
User manages resources
|
||||
Manage Resources UC3 City Data Publisher
|
||||
User tracks resources usage
|
||||
|
||||
Store City Data UC4
|
||||
2. City data is
|
||||
managed in a safe Transmit Data UC5 - Management Systems
|
||||
and intelligent
|
||||
manner
|
||||
Manage Infrastructure UC6
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 15
|
||||
|
||||
|
||||
City Data Publisher
|
||||
Set commercial city data
|
||||
Platform Provider
|
||||
Subscribe to proprietary data City Data Consumer
|
||||
Commercialise City
|
||||
UC7
|
||||
Data
|
||||
Manage commercial data City Data Publisher
|
||||
|
||||
3. City data is Manage data subscription City Data Consumer
|
||||
orchestrated in a
|
||||
Data Services Provider
|
||||
market place Set commercial data services
|
||||
Platform Provider
|
||||
Subscribe to commercial services Data Services Consumer
|
||||
Commercialise Data
|
||||
UC8
|
||||
Services
|
||||
Manage commercial services Data Services Provider
|
||||
|
||||
Manage services subscription Data Services Consumer
|
||||
UC9
|
||||
Register in the Platform City Data Consumer
|
||||
-
|
||||
4. City data is Discover City Data City Data Consumer
|
||||
UC10
|
||||
offered in an
|
||||
User consumes city data via data
|
||||
accessible manner
|
||||
API’s
|
||||
Consume City Data UC11 City Data Consumer
|
||||
User downloads datasets
|
||||
5. User’s Data Services Provider
|
||||
Deploy Data Services UC12
|
||||
experience is Platform Provider
|
||||
enhanced by the
|
||||
Manage Services UC13 - Data Services Provider
|
||||
provision of value-
|
||||
added services
|
||||
Utilise Data Services UC14 City Data Consumer
|
||||
|
||||
|
||||
|
||||
|
||||
2.3 SUB-GOAL 1: City data is collected in an intelligent manner
|
||||
Description: The urban platform enables the owners of city data to easily publish both historic and
|
||||
data streams in the platform, as well as their associated metadata.
|
||||
Rationale: This sub-goal is maintained by the services and functions to accept the publication of
|
||||
city data from data providers (of both open and proprietary data) and prepare the contents for
|
||||
storage and management within the urban platform. Functions include receiving data, performing
|
||||
quality assurance on data, verifying data formatting and document standards, associating meta-
|
||||
data information, and coordinating updates to databases and resources management.
|
||||
|
||||
Achieve [City data is exploited
|
||||
to its full effect]
|
||||
Goal
|
||||
co-ena bles
|
||||
|
||||
|
||||
|
||||
|
||||
How we achieve the platform overall goal?
|
||||
|
||||
|
||||
|
||||
Maintain [City data is provided in
|
||||
a harmonised manner]
|
||||
S ub-Goal 1
|
||||
|
||||
|
||||
|
||||
|
||||
What actions can maintain the sub-goal?
|
||||
|
||||
|
||||
|
||||
Achieve [Register Publisher] Achieve [Publish City Data] Achieve [Manage Resources]
|
||||
UC1 UC2 UC3
|
||||
|
||||
|
||||
|
||||
|
||||
Figure 5. Sub-Goal 1 “City data is collected in an intelligent manner” refinement.
|
||||
|
||||
|
||||
Drivers: Ensure data is published in an easy and uniform way.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 16
|
||||
|
||||
|
||||
Actions: For Sub-Goal 1 to be maintained in the long-run it requires the efficient realisation of use
|
||||
cases: “Publish City Data” and “Manage Resources”, as shown in Figure 5.
|
||||
|
||||
|
||||
2.3.1 Use Case: Register Publisher
|
||||
ID: UC1
|
||||
Refines: SUB-GOAL 1: City data is collected in an intelligent manner
|
||||
Pre-condition: Data Publisher is not logged in the system
|
||||
Actors: Data Publishers
|
||||
Rationale: Data Publishers can register in the platform and request approval to submit city data.
|
||||
They provide valid registration details (to be defined) and wait for registration confirmation.
|
||||
Platform Providers may authorise or not data publishers to offer both open and proprietary city data
|
||||
in the platform. Data submission agreement is a formal agreement between the Data Provider and
|
||||
the Urban Platform defining the terms of the content, standards, metadata creation, and license
|
||||
agreement. The Urban Platform will proactively work with Data Providers to agree on the content,
|
||||
quality and format of city data. Agreements between Platform and Data Providers may be
|
||||
renegotiated on a periodic or ad-hoc basis.
|
||||
Refines into requirements: FREQ.1 to FREQ.5.
|
||||
|
||||
Use Case Basic Stimulus and Responses
|
||||
1. The platform prompts the user for a username and password or
|
||||
register new account.
|
||||
2. The user selects registration option.
|
||||
3. The platform prompts user for publisher registration information (e.g.
|
||||
username, password, organisation)
|
||||
4. The user enters in their information.
|
||||
UC1. Register 5. Platform verifies information and creates account.
|
||||
Publisher o If non-valid information, platform shows error message and
|
||||
returns to step 1.
|
||||
6. Platform provider is requested to approve the account
|
||||
o Platform acknowledges registration has been successful
|
||||
o If non approved, platform shows error message and returns to
|
||||
step 1.
|
||||
7. End of registration
|
||||
|
||||
|
||||
2.3.2 Use Case: Publish City Data
|
||||
ID: UC2
|
||||
Refines: SUB-GOAL 1 - City data is collected in an intelligent manner
|
||||
Pre-condition: User is authenticated in the platform
|
||||
Actors: City data publisher
|
||||
Rationale: The Publish City Data function provides the appropriate mechanisms to receive city data from
|
||||
authorized data providers. Data may be manually uploaded or submitted via APIs. In general, data providers
|
||||
with whom the Urban Platform negotiates submission agreements are the providers of proprietary city data
|
||||
(those producing published material, i.e. publishers) and open data, and they can be either humans or
|
||||
machines. The providers of the Urban Platform will provide data providers with specifications on the content,
|
||||
quality and format of data, and publication terms and conditions.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 17
|
||||
|
||||
|
||||
The Publish City Data function may represent a legal transfer of custody for the data in the urban platform,
|
||||
and may require that special access controls be placed on the contents. This function provides a
|
||||
confirmation of receipt of data publication to the Producer, which may include a request to resubmit data in
|
||||
the case of errors resulting from the submission.
|
||||
Once data has arrived, it must undergo several reviews, including virus checking, format compliance,
|
||||
metadata minimum requirement agreement, quality and anticipated content and data formatting. The
|
||||
platform must include the ability to record all actions and decisions made concerning the publication of city
|
||||
data. The reasons for publication failure (e.g. missing metadata information, non-valid dataset) will be
|
||||
provided back to the city data publisher. In some cases, the provider can then resubmit corrected data and
|
||||
metadata information, while in other instances data publication refusal criteria should prevent the publisher
|
||||
from submitting the same dataset at a later time period (e.g. in cases of suspicious datasets – copyrights
|
||||
violation, viruses). When data is successfully submitted (either via APIs or manual upload), it will be
|
||||
processed/prepared for storage into the platform’s database.
|
||||
Specialised Use Cases: The Use Case Publish City Data data is distinguished into two
|
||||
specialised Use Cases: “User publishes city data via data API’s (UC2.1)” and “User manually
|
||||
uploads datasets (UC2.2)”.
|
||||
Subordinated Use Cases: “Store City Data (UC4)”
|
||||
Refines into requirements: FREQ 6 to FREQ.26.
|
||||
|
||||
|
||||
Specialised Use Cases Basic Stimulus and Responses
|
||||
1. Platform provides user with an interface for static data
|
||||
publication
|
||||
2. User selects datasets to be uploaded
|
||||
3. User provides metadata associated with the data (license,
|
||||
provenance, ownership, semantics) in accordance with defined
|
||||
standards
|
||||
UC2.1. User manually 4. User requests data publication
|
||||
uploads datasets 5. Platform quickly process user’s request for data publication
|
||||
6. Platform validates data submitted
|
||||
o If valid data, platform acknowledges data publication has
|
||||
been successful
|
||||
o If non-valid data, platform shows error message and returns
|
||||
to step 1.
|
||||
7. End of data publication
|
||||
1. Platform provides user with an interface for real-time data
|
||||
publication
|
||||
2. User input data API information
|
||||
3. User provides metadata associated with the data (license,
|
||||
provenance, ownership, semantics) in accordance with defined
|
||||
standards
|
||||
UC2.2. User publishes 4. User confirm information and request data publication
|
||||
city data via data API’s 5. Platform quickly process user’s request for data publication
|
||||
6. Platform validates data submitted
|
||||
o If valid data, platform acknowledges data publication has
|
||||
been successful
|
||||
o If non-valid data, platform shows error message and returns
|
||||
to step 1
|
||||
End of data publication
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 18
|
||||
|
||||
|
||||
2.3.3 Use Case: Manage Resources
|
||||
ID: UC3
|
||||
Refines: GOAL 1: City data is collected in an intelligent manner
|
||||
Pre-condition: User successfully authenticates in the platform
|
||||
Actors: City data publisher
|
||||
Rationale: Manage resources provides the services and functions for updating, maintaining and
|
||||
accessing both data and metadata, as well as tracking the usage of resources by users. Ideally
|
||||
the owners of the resources should be the only authorised user to manage resources, and other
|
||||
authorised users can track the usage of the resources in the platform. The platform must provide a
|
||||
database update response indicating the status of the update, avoid update errors to be
|
||||
propagated in the platform, and should keep an audit trail of all actions to enable rollback. Data
|
||||
usage tracking includes performing queries on the data management data to generate result sets,
|
||||
and producing reports from these result sets.
|
||||
Specialised Use Cases: The Use Case Manage Resources data is distinguished into two
|
||||
specialised Use Cases: “User manages resources (UC3.1)” and “User tracks resources usage
|
||||
(UC3.2)”.
|
||||
Subordinated Use Cases: “Transmit Data (UC5)”
|
||||
Refines into requirements: FREQ.27 to FREQ.25.
|
||||
|
||||
|
||||
Specialised Use Cases Basic Interactions and Responses
|
||||
1. Platform provides user with an interface for resources
|
||||
management (e.g. data and metadata, data usage)
|
||||
2. User chooses to edit or delete data
|
||||
3. If edit, user revise metadata associated with the data (license,
|
||||
provenance, ownership, access-control, semantics);
|
||||
UC3.1. User manages 4. If delete, user selects dataset(s) to be removed
|
||||
resources 5. User confirms action
|
||||
6. Platform quickly process user’s request
|
||||
7. Platform confirms execution of request
|
||||
o If valid request, platform acknowledges request has been
|
||||
processed successfully
|
||||
o If non-valid request, platform returns to step 1.
|
||||
8. End of resources management
|
||||
1. Platform provides user with an interface for resources
|
||||
management (e.g. data and metadata, data usage)
|
||||
2. User chooses to visualise usage information of a dataset
|
||||
UC3.2. User tracks 3. Platform quickly process user’s request for data usage
|
||||
resources usage information
|
||||
4. Platform provides user with statistical information about data
|
||||
usage and data users anonymised information
|
||||
5. End of data usage tracking.
|
||||
|
||||
|
||||
2.3.4 Functional Requirements
|
||||
|
||||
Req. ID UC. ID Description Priority Domain
|
||||
Societal Needs,
|
||||
FREQ.1 UC1 Allow data publishers to register to submit data for publication Must
|
||||
Platform
|
||||
Tracks data publication agreements between Data and Platform Business Needs,
|
||||
FREQ.2 UC1 Must
|
||||
Providers Platform
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 19
|
||||
|
||||
|
||||
Store terms of agreements, and use them to monitor/review/process
|
||||
FREQ.3 UC1 Must City Data, Platform
|
||||
data submissions.
|
||||
Able to add and edit terms of agreement, based on access of level of Business Needs,
|
||||
FREQ.4 UC1 Must
|
||||
user. Platform
|
||||
|
||||
FREQ.5 UC1 Data publications are managed and monitored Must City Data, Platform
|
||||
|
||||
Allow authenticated users from across different organisations to City Data, Platform,
|
||||
FREQ.6 UC2 Must
|
||||
publish city data Business Needs
|
||||
Provide authorization mechanisms for users and sensors to publish
|
||||
FREQ.7 UC2 Must City Data, Platform
|
||||
city data
|
||||
City Data, Platform,
|
||||
FREQ.9 UC2 Provide mechanisms for static data publication Must
|
||||
Business Needs
|
||||
City Data, Platform,
|
||||
FREQ.10 UC2 Provide mechanisms for real-time data publication Must
|
||||
Business Needs
|
||||
|
||||
FREQ.11 UC2 Enable the publication of metadata Must City Data, Platform
|
||||
|
||||
|
||||
FREQ.12 UC2 Maintain temporal information about the data Must City Data, Platform
|
||||
|
||||
|
||||
FREQ.13 UC2 Support sensory data collection Must City Data, Platform
|
||||
|
||||
|
||||
FREQ.14 UC2 Accepts content in numerous file types/formats Must City Data, Platform
|
||||
|
||||
Prompts a request for resubmission to the data provider if an error of
|
||||
600
files/core_requirements_part3.txt
Normal file
600
files/core_requirements_part3.txt
Normal file
@@ -0,0 +1,600 @@
|
||||
harmonised way
|
||||
full effect
|
||||
|
||||
|
||||
|
||||
|
||||
User manages resources
|
||||
Manage Resources UC3 City Data Publisher
|
||||
User tracks resources usage
|
||||
|
||||
Store City Data UC4
|
||||
2. City data is
|
||||
managed in a safe Transmit Data UC5 - Management Systems
|
||||
and intelligent
|
||||
manner
|
||||
Manage Infrastructure UC6
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 15
|
||||
|
||||
|
||||
City Data Publisher
|
||||
Set commercial city data
|
||||
Platform Provider
|
||||
Subscribe to proprietary data City Data Consumer
|
||||
Commercialise City
|
||||
UC7
|
||||
Data
|
||||
Manage commercial data City Data Publisher
|
||||
|
||||
3. City data is Manage data subscription City Data Consumer
|
||||
orchestrated in a
|
||||
Data Services Provider
|
||||
market place Set commercial data services
|
||||
Platform Provider
|
||||
Subscribe to commercial services Data Services Consumer
|
||||
Commercialise Data
|
||||
UC8
|
||||
Services
|
||||
Manage commercial services Data Services Provider
|
||||
|
||||
Manage services subscription Data Services Consumer
|
||||
UC9
|
||||
Register in the Platform City Data Consumer
|
||||
-
|
||||
4. City data is Discover City Data City Data Consumer
|
||||
UC10
|
||||
offered in an
|
||||
User consumes city data via data
|
||||
accessible manner
|
||||
API’s
|
||||
Consume City Data UC11 City Data Consumer
|
||||
User downloads datasets
|
||||
5. User’s Data Services Provider
|
||||
Deploy Data Services UC12
|
||||
experience is Platform Provider
|
||||
enhanced by the
|
||||
Manage Services UC13 - Data Services Provider
|
||||
provision of value-
|
||||
added services
|
||||
Utilise Data Services UC14 City Data Consumer
|
||||
|
||||
|
||||
|
||||
|
||||
2.3 SUB-GOAL 1: City data is collected in an intelligent manner
|
||||
Description: The urban platform enables the owners of city data to easily publish both historic and
|
||||
data streams in the platform, as well as their associated metadata.
|
||||
Rationale: This sub-goal is maintained by the services and functions to accept the publication of
|
||||
city data from data providers (of both open and proprietary data) and prepare the contents for
|
||||
storage and management within the urban platform. Functions include receiving data, performing
|
||||
quality assurance on data, verifying data formatting and document standards, associating meta-
|
||||
data information, and coordinating updates to databases and resources management.
|
||||
|
||||
Achieve [City data is exploited
|
||||
to its full effect]
|
||||
Goal
|
||||
co-ena bles
|
||||
|
||||
|
||||
|
||||
|
||||
How we achieve the platform overall goal?
|
||||
|
||||
|
||||
|
||||
Maintain [City data is provided in
|
||||
a harmonised manner]
|
||||
S ub-Goal 1
|
||||
|
||||
|
||||
|
||||
|
||||
What actions can maintain the sub-goal?
|
||||
|
||||
|
||||
|
||||
Achieve [Register Publisher] Achieve [Publish City Data] Achieve [Manage Resources]
|
||||
UC1 UC2 UC3
|
||||
|
||||
|
||||
|
||||
|
||||
Figure 5. Sub-Goal 1 “City data is collected in an intelligent manner” refinement.
|
||||
|
||||
|
||||
Drivers: Ensure data is published in an easy and uniform way.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 16
|
||||
|
||||
|
||||
Actions: For Sub-Goal 1 to be maintained in the long-run it requires the efficient realisation of use
|
||||
cases: “Publish City Data” and “Manage Resources”, as shown in Figure 5.
|
||||
|
||||
|
||||
2.3.1 Use Case: Register Publisher
|
||||
ID: UC1
|
||||
Refines: SUB-GOAL 1: City data is collected in an intelligent manner
|
||||
Pre-condition: Data Publisher is not logged in the system
|
||||
Actors: Data Publishers
|
||||
Rationale: Data Publishers can register in the platform and request approval to submit city data.
|
||||
They provide valid registration details (to be defined) and wait for registration confirmation.
|
||||
Platform Providers may authorise or not data publishers to offer both open and proprietary city data
|
||||
in the platform. Data submission agreement is a formal agreement between the Data Provider and
|
||||
the Urban Platform defining the terms of the content, standards, metadata creation, and license
|
||||
agreement. The Urban Platform will proactively work with Data Providers to agree on the content,
|
||||
quality and format of city data. Agreements between Platform and Data Providers may be
|
||||
renegotiated on a periodic or ad-hoc basis.
|
||||
Refines into requirements: FREQ.1 to FREQ.5.
|
||||
|
||||
Use Case Basic Stimulus and Responses
|
||||
1. The platform prompts the user for a username and password or
|
||||
register new account.
|
||||
2. The user selects registration option.
|
||||
3. The platform prompts user for publisher registration information (e.g.
|
||||
username, password, organisation)
|
||||
4. The user enters in their information.
|
||||
UC1. Register 5. Platform verifies information and creates account.
|
||||
Publisher o If non-valid information, platform shows error message and
|
||||
returns to step 1.
|
||||
6. Platform provider is requested to approve the account
|
||||
o Platform acknowledges registration has been successful
|
||||
o If non approved, platform shows error message and returns to
|
||||
step 1.
|
||||
7. End of registration
|
||||
|
||||
|
||||
2.3.2 Use Case: Publish City Data
|
||||
ID: UC2
|
||||
Refines: SUB-GOAL 1 - City data is collected in an intelligent manner
|
||||
Pre-condition: User is authenticated in the platform
|
||||
Actors: City data publisher
|
||||
Rationale: The Publish City Data function provides the appropriate mechanisms to receive city data from
|
||||
authorized data providers. Data may be manually uploaded or submitted via APIs. In general, data providers
|
||||
with whom the Urban Platform negotiates submission agreements are the providers of proprietary city data
|
||||
(those producing published material, i.e. publishers) and open data, and they can be either humans or
|
||||
machines. The providers of the Urban Platform will provide data providers with specifications on the content,
|
||||
quality and format of data, and publication terms and conditions.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 17
|
||||
|
||||
|
||||
The Publish City Data function may represent a legal transfer of custody for the data in the urban platform,
|
||||
and may require that special access controls be placed on the contents. This function provides a
|
||||
confirmation of receipt of data publication to the Producer, which may include a request to resubmit data in
|
||||
the case of errors resulting from the submission.
|
||||
Once data has arrived, it must undergo several reviews, including virus checking, format compliance,
|
||||
metadata minimum requirement agreement, quality and anticipated content and data formatting. The
|
||||
platform must include the ability to record all actions and decisions made concerning the publication of city
|
||||
data. The reasons for publication failure (e.g. missing metadata information, non-valid dataset) will be
|
||||
provided back to the city data publisher. In some cases, the provider can then resubmit corrected data and
|
||||
metadata information, while in other instances data publication refusal criteria should prevent the publisher
|
||||
from submitting the same dataset at a later time period (e.g. in cases of suspicious datasets – copyrights
|
||||
violation, viruses). When data is successfully submitted (either via APIs or manual upload), it will be
|
||||
processed/prepared for storage into the platform’s database.
|
||||
Specialised Use Cases: The Use Case Publish City Data data is distinguished into two
|
||||
specialised Use Cases: “User publishes city data via data API’s (UC2.1)” and “User manually
|
||||
uploads datasets (UC2.2)”.
|
||||
Subordinated Use Cases: “Store City Data (UC4)”
|
||||
Refines into requirements: FREQ 6 to FREQ.26.
|
||||
|
||||
|
||||
Specialised Use Cases Basic Stimulus and Responses
|
||||
1. Platform provides user with an interface for static data
|
||||
publication
|
||||
2. User selects datasets to be uploaded
|
||||
3. User provides metadata associated with the data (license,
|
||||
provenance, ownership, semantics) in accordance with defined
|
||||
standards
|
||||
UC2.1. User manually 4. User requests data publication
|
||||
uploads datasets 5. Platform quickly process user’s request for data publication
|
||||
6. Platform validates data submitted
|
||||
o If valid data, platform acknowledges data publication has
|
||||
been successful
|
||||
o If non-valid data, platform shows error message and returns
|
||||
to step 1.
|
||||
7. End of data publication
|
||||
1. Platform provides user with an interface for real-time data
|
||||
publication
|
||||
2. User input data API information
|
||||
3. User provides metadata associated with the data (license,
|
||||
provenance, ownership, semantics) in accordance with defined
|
||||
standards
|
||||
UC2.2. User publishes 4. User confirm information and request data publication
|
||||
city data via data API’s 5. Platform quickly process user’s request for data publication
|
||||
6. Platform validates data submitted
|
||||
o If valid data, platform acknowledges data publication has
|
||||
been successful
|
||||
o If non-valid data, platform shows error message and returns
|
||||
to step 1
|
||||
End of data publication
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 18
|
||||
|
||||
|
||||
2.3.3 Use Case: Manage Resources
|
||||
ID: UC3
|
||||
Refines: GOAL 1: City data is collected in an intelligent manner
|
||||
Pre-condition: User successfully authenticates in the platform
|
||||
Actors: City data publisher
|
||||
Rationale: Manage resources provides the services and functions for updating, maintaining and
|
||||
accessing both data and metadata, as well as tracking the usage of resources by users. Ideally
|
||||
the owners of the resources should be the only authorised user to manage resources, and other
|
||||
authorised users can track the usage of the resources in the platform. The platform must provide a
|
||||
database update response indicating the status of the update, avoid update errors to be
|
||||
propagated in the platform, and should keep an audit trail of all actions to enable rollback. Data
|
||||
usage tracking includes performing queries on the data management data to generate result sets,
|
||||
and producing reports from these result sets.
|
||||
Specialised Use Cases: The Use Case Manage Resources data is distinguished into two
|
||||
specialised Use Cases: “User manages resources (UC3.1)” and “User tracks resources usage
|
||||
(UC3.2)”.
|
||||
Subordinated Use Cases: “Transmit Data (UC5)”
|
||||
Refines into requirements: FREQ.27 to FREQ.25.
|
||||
|
||||
|
||||
Specialised Use Cases Basic Interactions and Responses
|
||||
1. Platform provides user with an interface for resources
|
||||
management (e.g. data and metadata, data usage)
|
||||
2. User chooses to edit or delete data
|
||||
3. If edit, user revise metadata associated with the data (license,
|
||||
provenance, ownership, access-control, semantics);
|
||||
UC3.1. User manages 4. If delete, user selects dataset(s) to be removed
|
||||
resources 5. User confirms action
|
||||
6. Platform quickly process user’s request
|
||||
7. Platform confirms execution of request
|
||||
o If valid request, platform acknowledges request has been
|
||||
processed successfully
|
||||
o If non-valid request, platform returns to step 1.
|
||||
8. End of resources management
|
||||
1. Platform provides user with an interface for resources
|
||||
management (e.g. data and metadata, data usage)
|
||||
2. User chooses to visualise usage information of a dataset
|
||||
UC3.2. User tracks 3. Platform quickly process user’s request for data usage
|
||||
resources usage information
|
||||
4. Platform provides user with statistical information about data
|
||||
usage and data users anonymised information
|
||||
5. End of data usage tracking.
|
||||
|
||||
|
||||
2.3.4 Functional Requirements
|
||||
|
||||
Req. ID UC. ID Description Priority Domain
|
||||
Societal Needs,
|
||||
FREQ.1 UC1 Allow data publishers to register to submit data for publication Must
|
||||
Platform
|
||||
Tracks data publication agreements between Data and Platform Business Needs,
|
||||
FREQ.2 UC1 Must
|
||||
Providers Platform
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 19
|
||||
|
||||
|
||||
Store terms of agreements, and use them to monitor/review/process
|
||||
FREQ.3 UC1 Must City Data, Platform
|
||||
data submissions.
|
||||
Able to add and edit terms of agreement, based on access of level of Business Needs,
|
||||
FREQ.4 UC1 Must
|
||||
user. Platform
|
||||
|
||||
FREQ.5 UC1 Data publications are managed and monitored Must City Data, Platform
|
||||
|
||||
Allow authenticated users from across different organisations to City Data, Platform,
|
||||
FREQ.6 UC2 Must
|
||||
publish city data Business Needs
|
||||
Provide authorization mechanisms for users and sensors to publish
|
||||
FREQ.7 UC2 Must City Data, Platform
|
||||
city data
|
||||
City Data, Platform,
|
||||
FREQ.9 UC2 Provide mechanisms for static data publication Must
|
||||
Business Needs
|
||||
City Data, Platform,
|
||||
FREQ.10 UC2 Provide mechanisms for real-time data publication Must
|
||||
Business Needs
|
||||
|
||||
FREQ.11 UC2 Enable the publication of metadata Must City Data, Platform
|
||||
|
||||
|
||||
FREQ.12 UC2 Maintain temporal information about the data Must City Data, Platform
|
||||
|
||||
|
||||
FREQ.13 UC2 Support sensory data collection Must City Data, Platform
|
||||
|
||||
|
||||
FREQ.14 UC2 Accepts content in numerous file types/formats Must City Data, Platform
|
||||
|
||||
Prompts a request for resubmission to the data provider if an error of
|
||||
FREQ.15 UC2 Must City Data, Platform
|
||||
data transmission or receipt occurs
|
||||
|
||||
FREQ.16 UC2 Enable the semantic description of connected devices Must City Data, Platform
|
||||
|
||||
|
||||
FREQ.17 UC2 Gather data from authenticated and authorized devices Must City Data, Platform
|
||||
|
||||
|
||||
FREQ.18 UC2 Validates automatically the successful transfer of the data Must City Data, Platform
|
||||
|
||||
|
||||
FREQ.19 UC2 Performs virus checking on data Must City Data, Platform
|
||||
|
||||
Verifies the validity of the submission based on submitter, expected
|
||||
FREQ.20 UC2 Must City Data, Platform
|
||||
format, data quality, and completeness
|
||||
Platform should have built-in checks on the incoming metadata. Data
|
||||
FREQ.21 UC2 not containing the minimally defined set of attributes should be Must City Data, Platform
|
||||
returned to the publisher for metadata enhancement.
|
||||
System should have a user-friendly method of mapping non-standard
|
||||
FREQ.22 UC2 Should City Data, Platform
|
||||
metadata elements into approved standard elements.
|
||||
Once ingested, metadata should be stored in a single common format.
|
||||
FREQ.23 UC2 This format should be one that ensures against data loss, and allows a Must City Data, Platform
|
||||
variety of access/distribution options
|
||||
Data in the repository shall have sufficient technical metadata to
|
||||
FREQ.24 UC2 assure functionality (e.g. viewing and display) to ensure accessibility Must City Data, Platform
|
||||
and reusability.
|
||||
Allows publishers to display and perform manual/visual quality control Business Needs, City
|
||||
FREQ.25 UC2 Could
|
||||
assurance via a user-friendly GUI Data, Platform
|
||||
Business Needs, City
|
||||
FREQ.26 UC2 Any errors shall prompt a request for resubmission of data Should
|
||||
Data, Platform
|
||||
|
||||
FREQ.27 UC3 Enable data providers to manage their resources Must Business Needs
|
||||
|
||||
A minimal set of identifying information/metadata concerning data Business Needs,
|
||||
FREQ.28 UC3 Must
|
||||
publication submission must be recorded Platform
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 20
|
||||
|
||||
|
||||
Stores and tracks versions of data. Links /connections between
|
||||
FREQ.29 UC3 Must City Data, Platform
|
||||
versions are created and maintained
|
||||
Give service and data providers access to anonymized data of the Business Needs
|
||||
FREQ.30 UC3 Should
|
||||
subscribers of their data or services
|
||||
City Data, Platform,
|
||||
FREQ.31 UC3 Enable data providers to maintain and repair data and metadata Should
|
||||
Business Needs
|
||||
Tracks data publication agreements between Data and Platform Business Needs,
|
||||
FREQ.32 UC3 Must
|
||||
Providers Platform
|
||||
Store terms of agreements, and use them to monitor/review/process
|
||||
FREQ.33 UC3 Must City Data, Platform
|
||||
data submissions.
|
||||
Able to add and edit terms of agreement, based on access of level of Business Needs,
|
||||
FREQ.34 UC3 Must
|
||||
user. Platform
|
||||
|
||||
FREQ.35 UC3 Submission volumes and schedules are managed and monitored Must City Data, Platform
|
||||
|
||||
|
||||
|
||||
|
||||
2.4 SUB-GOAL 2: City data is managed in a safe and intelligent manner
|
||||
Rationale: The urban platform enables users to publish, consume and commercialise data, as well
|
||||
as deploy and manage services all in a secure and privacy protected manner.
|
||||
|
||||
Achieve [City data is exploited
|
||||
to its full effect]
|
||||
Goal
|
||||
co-ena bles
|
||||
|
||||
|
||||
|
||||
|
||||
Maintain [Resources are managed in a
|
||||
safe and intelligent manner]
|
||||
S ub-Goal 1
|
||||
|
||||
|
||||
|
||||
|
||||
Achieve [Store Data] Achieve [Transmit Data] Achieve [Manage Infrastructure]
|
||||
UC4 UC5 UC6
|
||||
|
||||
|
||||
Figure 6. Sub-Goal 2 “City data is managed in a safe and intelligent manner” refinement.
|
||||
|
||||
|
||||
|
||||
Drivers: Ensure data is secured and the identity of users are preserved
|
||||
Actions: For Sub-Goal 2 to be maintained in the long-run it requires the efficient realisation of use
|
||||
cases: “Store City Data” and “Retrieve and Transmit City Data”, as shown in Figure 6.
|
||||
|
||||
|
||||
2.4.1 Use Case: Store City Data
|
||||
ID: UC4
|
||||
Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner
|
||||
Pre-condition: Data is successfully published in the platform
|
||||
Actors: Urban Platform
|
||||
Rationale: When data is successfully submitted (either via APIs or manual upload), it is
|
||||
processed/prepared for storage into the platform’s database. This procedure will include the
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 21
|
||||
|
||||
|
||||
generation of unique identifiers to the database, enrichment with ontologies (when applicable),
|
||||
encrypted (when applicable), signed with digital certificates (when applicable) to ensure that the
|
||||
data conforms to the platform data formatting, standards, security and regulation. Data may be
|
||||
converted to accepted formats, as needed (e.g. graph model). A primary goal of the conversion of
|
||||
content for the platform is the preservation of the content. Priority will be given to preserving the
|
||||
data accordingly to the policies defined in section (2.6.2). Access-control levels and license models
|
||||
are associated to data which is subject to restrictions relating to access and conditions of use.
|
||||
Refines into requirements: FREQ 28 – FREQ 39.
|
||||
|
||||
Use Case Basic Interactions and Responses
|
||||
1. Platform mechanisms converts submitted data into a standard format
|
||||
2. Security enforcement (e.g. anonymisation, cryptography) is placed on
|
||||
sensitive information.
|
||||
UC4. Store City 3. Platform associates with datasets the access-control definitions set by
|
||||
Data owner of resources.
|
||||
4. Data is enriched with semantics and is associated with other datasets
|
||||
5. Platform stores data in a scalable and secure database.
|
||||
6. End of data storage.
|
||||
|
||||
|
||||
2.4.2 Use Case: Transmit Data
|
||||
ID: UC5
|
||||
Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner
|
||||
Pre-condition: Data is successfully published in the platform
|
||||
Actors: Urban Platform
|
||||
Rationale: The platform accepts data retrieval request, validates user’s rights to access the data,
|
||||
retrieves city data from data storage, and moves a copy of the data to the relevant platform
|
||||
component for further processing. If special processing is required, the retrieval function accesses
|
||||
data in staging storage and applies the requested processes. The types of operations, which may
|
||||
be carried out, include sub-sampling in temporal or spatial dimensions, conversions between
|
||||
different data types or output formats, and other specialized processing (e.g., data visualisation).
|
||||
Once it is finalised data will be sent to the appropriate delivery channels (e.g. API’s, GUI). It also
|
||||
encompasses function to verify corruption during any internal data transfer. This function requires
|
||||
that all hardware and software within the platform provide notification of potential errors and that
|
||||
these errors are routed to standard error logs that are checked by the Platform Provider.
|
||||
Refines into requirements: FREQ 40 to 54.
|
||||
2.4.3 Use Case: Manage Infrastructure
|
||||
ID: UC6
|
||||
Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner
|
||||
Pre-condition: The platform is available
|
||||
Actors: Urban Platform
|
||||
Rationale: Manage infrastructure provides the services and functions for the overall operation of
|
||||
the urban platform. Administration functions include monitoring quality of service agreements,
|
||||
auditing data publication to ensure that they meet archive standards, and maintaining configuration
|
||||
management of system hardware and software. In overall, it provides system engineering
|
||||
functions to monitor and improve platform operations, and to inventory, report on, and
|
||||
migrate/update the contents of the platforms’ databases.
|
||||
Refines into requirements: FREQ 55 to 62.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 22
|
||||
|
||||
|
||||
|
||||
Use Case Basic Interactions and Responses
|
||||
1. Platform keeps monitoring services at run-time to ensure operation
|
||||
and integrity of city data
|
||||
o If system failure, the platform activates mechanisms for recovery
|
||||
UC6. Manage based on pre-defined rules
|
||||
Infrastructure o Platform logs issue and issue alert messages to platform providers
|
||||
2. Platform logs operation capabilities (e.g. performance, mean of time
|
||||
failure, issues, etc.)
|
||||
|
||||
|
||||
2.4.4 Functional Requirements
|
||||
|
||||
Req. ID UC. ID Description Priority Domain
|
||||
A minimal set of identifying information/metadata concerning data Business Needs,
|
||||
FREQ.28 UC4 Must
|
||||
publication submission must be recorded. Platform
|
||||
Stores and tracks versions of data. Links /connections between
|
||||
FREQ.29 UC4 Must City Data, Platform
|
||||
versions are created and maintained.
|
||||
|
||||
FREQ.30 UC4 Converts data to accepted file formats Must City Data, Platform
|
||||
|
||||
Keep sensitive information secured and accessible only to authorized
|
||||
FREQ.31 UC4 Must City Data, Platform
|
||||
users
|
||||
|
||||
FREQ.32 UC4 Keep user’s personal information protected Should City Data, Platform
|
||||
|
||||
|
||||
FREQ.33 UC4 Keep city data and meta-data secured Must Platform Needs
|
||||
|
||||
|
||||
FREQ.34 UC4 Enable privacy preserving mechanisms associated to data Must Platform Needs
|
||||
|
||||
|
||||
FREQ.35 UC4 Model data in accordance with defined standards Must City Data, Platform
|
||||
|
||||
|
||||
FREQ.36 UC4 Support the use of ontologies and semantic modelling of city data Could City Data
|
||||
|
||||
|
||||
FREQ.37 UC4 Support database-level provenance annotation Should City Data
|
||||
|
||||
|
||||
FREQ.38 UC4 Support data-level provenance annotation Should City Data
|
||||
|
||||
|
||||
FREQ.39 UC4 Enable data to be encrypted Should Platform Needs
|
||||
|
||||
System must have the ability to search and display metadata, preferably
|
||||
City Data, Platform
|
||||
FREQ.40 UC5 in a user-conformable, human readable display as well as in its native Must
|
||||
Needs
|
||||
format for machine harvesting and manipulation.
|
||||
Controls access to data in the repository based on multiple permission
|
||||
City Data, Platform
|
||||
FREQ.41 UC5 levels. These permission levels determine the create/edit/read/delete Should
|
||||
Needs
|
||||
privileges granted users.
|
||||
Access rights and conditions of use will be held for each data and its City Data, Platform
|
||||
FREQ.42 UC5 Should
|
||||
related metadata. Needs
|
||||
Access rights and conditions can be inherited from a parent data to any City Data, Platform
|
||||
FREQ.43 UC5 Could
|
||||
data designated as a child data (derived information). Needs
|
||||
Access rights and conditions of use will be machine readable and City Data, Platform
|
||||
FREQ.44 UC5 Should
|
||||
actionable. Needs
|
||||
Access mechanisms must be sufficiently granular to allow the Business Needs, City
|
||||
FREQ.45 UC5 identification of individual users, in order to maintain audit logs of actions Should
|
||||
Data, Platform
|
||||
performed by users.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 23
|
||||
|
||||
|
||||
Maintains the integrity of the database which contains both metadata
|
||||
FREQ.46 UC5 and system information. Must Platform Needs
|
||||
|
||||
Provides internal validation such as referential integrity of the contents of City Data, Platform
|
||||
FREQ.47 UC5 the database. Must
|
||||
Needs
|
||||
Creates and maintains schema definitions required to support data
|
||||
FREQ.48 UC5 management functions. Must Platform Needs
|
||||
|
||||
Monitors and ensures that data and metadata are not corrupted during
|
||||
FREQ.49 UC5 transfers. Must Platform Needs
|
||||
|
||||
Provides statistically acceptable assurance that no components of the
|
||||
FREQ.50 UC5 data are corrupted during any internal data transfer. Must Platform Needs
|
||||
|
||||
Performs routine and special data integrity checking for each dataset
|
||||
FREQ.51 UC5 and generates error reports. Must Platform Needs
|
||||
|
||||
Provides disaster recovery capabilities including data backup, off-site
|
||||
FREQ.52 UC5 data storage, data recovery, etc. Must Platform Needs
|
||||
|
||||
Refresh/replace data without service interruption, and update
|
||||
FREQ.53 UC5 corresponding metadata as appropriate. Must Platform Needs
|
||||
|
||||
Ensure that any associated unique identifiers of the updated data are not
|
||||
FREQ.54 UC5 altered. Must Platform Needs
|
||||
|
||||
Audits submissions to ensure that they meet archive/repository
|
||||
FREQ.55 UC6 standards. Must Platform Needs
|
||||
|
||||
Maintains configuration management of the system hardware and
|
||||
FREQ.56 UC6 software. Must Platform Needs
|
||||
|
||||
Has capability to inventory, report on and migrate the contents of the
|
||||
FREQ.57 UC6 repository. Must Platform Needs
|
||||
|
||||
|
||||
FREQ.58 UC6 Ensures data integrity for version upgrades and format migration. Must Platform Needs
|
||||
|
||||
|
||||
FREQ.59 UC6 Monitors functionality of the entire repository. Must Platform Needs
|
||||
|
||||
|
||||
FREQ.60 UC6 Maintains integrity of system configuration. Must Platform Needs
|
||||
|
||||
Audits system operations, performance and usage.
|
||||
FREQ.61 UC6 Must Platform Needs
|
||||
|
||||
Provides platform performance information and database holdings
|
||||
FREQ.62 UC6 inventory reports Must Platform Needs
|
||||
|
||||
|
||||
|
||||
|
||||
2.5 SUB-GOAL 3: City data is orchestrated in a marketplace
|
||||
Rationale: The urban platform enables users to consume and publish data in a secure and privacy
|
||||
protected manner.
|
||||
Drivers: Ensure data is secured and the identity of users are preserved
|
||||
Actions: For Sub-Goal 3 to be maintained in the long-run it requires the efficient realisation of use
|
||||
cases: “Commercialise City Data” and “Commercialise Data Services”, as shown in Figure 7.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 24
|
||||
|
||||
|
||||
700
files/core_requirements_part4.txt
Normal file
700
files/core_requirements_part4.txt
Normal file
@@ -0,0 +1,700 @@
|
||||
city data via data API’s 5. Platform quickly process user’s request for data publication
|
||||
6. Platform validates data submitted
|
||||
o If valid data, platform acknowledges data publication has
|
||||
been successful
|
||||
o If non-valid data, platform shows error message and returns
|
||||
to step 1
|
||||
End of data publication
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 18
|
||||
|
||||
|
||||
2.3.3 Use Case: Manage Resources
|
||||
ID: UC3
|
||||
Refines: GOAL 1: City data is collected in an intelligent manner
|
||||
Pre-condition: User successfully authenticates in the platform
|
||||
Actors: City data publisher
|
||||
Rationale: Manage resources provides the services and functions for updating, maintaining and
|
||||
accessing both data and metadata, as well as tracking the usage of resources by users. Ideally
|
||||
the owners of the resources should be the only authorised user to manage resources, and other
|
||||
authorised users can track the usage of the resources in the platform. The platform must provide a
|
||||
database update response indicating the status of the update, avoid update errors to be
|
||||
propagated in the platform, and should keep an audit trail of all actions to enable rollback. Data
|
||||
usage tracking includes performing queries on the data management data to generate result sets,
|
||||
and producing reports from these result sets.
|
||||
Specialised Use Cases: The Use Case Manage Resources data is distinguished into two
|
||||
specialised Use Cases: “User manages resources (UC3.1)” and “User tracks resources usage
|
||||
(UC3.2)”.
|
||||
Subordinated Use Cases: “Transmit Data (UC5)”
|
||||
Refines into requirements: FREQ.27 to FREQ.25.
|
||||
|
||||
|
||||
Specialised Use Cases Basic Interactions and Responses
|
||||
1. Platform provides user with an interface for resources
|
||||
management (e.g. data and metadata, data usage)
|
||||
2. User chooses to edit or delete data
|
||||
3. If edit, user revise metadata associated with the data (license,
|
||||
provenance, ownership, access-control, semantics);
|
||||
UC3.1. User manages 4. If delete, user selects dataset(s) to be removed
|
||||
resources 5. User confirms action
|
||||
6. Platform quickly process user’s request
|
||||
7. Platform confirms execution of request
|
||||
o If valid request, platform acknowledges request has been
|
||||
processed successfully
|
||||
o If non-valid request, platform returns to step 1.
|
||||
8. End of resources management
|
||||
1. Platform provides user with an interface for resources
|
||||
management (e.g. data and metadata, data usage)
|
||||
2. User chooses to visualise usage information of a dataset
|
||||
UC3.2. User tracks 3. Platform quickly process user’s request for data usage
|
||||
resources usage information
|
||||
4. Platform provides user with statistical information about data
|
||||
usage and data users anonymised information
|
||||
5. End of data usage tracking.
|
||||
|
||||
|
||||
2.3.4 Functional Requirements
|
||||
|
||||
Req. ID UC. ID Description Priority Domain
|
||||
Societal Needs,
|
||||
FREQ.1 UC1 Allow data publishers to register to submit data for publication Must
|
||||
Platform
|
||||
Tracks data publication agreements between Data and Platform Business Needs,
|
||||
FREQ.2 UC1 Must
|
||||
Providers Platform
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 19
|
||||
|
||||
|
||||
Store terms of agreements, and use them to monitor/review/process
|
||||
FREQ.3 UC1 Must City Data, Platform
|
||||
data submissions.
|
||||
Able to add and edit terms of agreement, based on access of level of Business Needs,
|
||||
FREQ.4 UC1 Must
|
||||
user. Platform
|
||||
|
||||
FREQ.5 UC1 Data publications are managed and monitored Must City Data, Platform
|
||||
|
||||
Allow authenticated users from across different organisations to City Data, Platform,
|
||||
FREQ.6 UC2 Must
|
||||
publish city data Business Needs
|
||||
Provide authorization mechanisms for users and sensors to publish
|
||||
FREQ.7 UC2 Must City Data, Platform
|
||||
city data
|
||||
City Data, Platform,
|
||||
FREQ.9 UC2 Provide mechanisms for static data publication Must
|
||||
Business Needs
|
||||
City Data, Platform,
|
||||
FREQ.10 UC2 Provide mechanisms for real-time data publication Must
|
||||
Business Needs
|
||||
|
||||
FREQ.11 UC2 Enable the publication of metadata Must City Data, Platform
|
||||
|
||||
|
||||
FREQ.12 UC2 Maintain temporal information about the data Must City Data, Platform
|
||||
|
||||
|
||||
FREQ.13 UC2 Support sensory data collection Must City Data, Platform
|
||||
|
||||
|
||||
FREQ.14 UC2 Accepts content in numerous file types/formats Must City Data, Platform
|
||||
|
||||
Prompts a request for resubmission to the data provider if an error of
|
||||
FREQ.15 UC2 Must City Data, Platform
|
||||
data transmission or receipt occurs
|
||||
|
||||
FREQ.16 UC2 Enable the semantic description of connected devices Must City Data, Platform
|
||||
|
||||
|
||||
FREQ.17 UC2 Gather data from authenticated and authorized devices Must City Data, Platform
|
||||
|
||||
|
||||
FREQ.18 UC2 Validates automatically the successful transfer of the data Must City Data, Platform
|
||||
|
||||
|
||||
FREQ.19 UC2 Performs virus checking on data Must City Data, Platform
|
||||
|
||||
Verifies the validity of the submission based on submitter, expected
|
||||
FREQ.20 UC2 Must City Data, Platform
|
||||
format, data quality, and completeness
|
||||
Platform should have built-in checks on the incoming metadata. Data
|
||||
FREQ.21 UC2 not containing the minimally defined set of attributes should be Must City Data, Platform
|
||||
returned to the publisher for metadata enhancement.
|
||||
System should have a user-friendly method of mapping non-standard
|
||||
FREQ.22 UC2 Should City Data, Platform
|
||||
metadata elements into approved standard elements.
|
||||
Once ingested, metadata should be stored in a single common format.
|
||||
FREQ.23 UC2 This format should be one that ensures against data loss, and allows a Must City Data, Platform
|
||||
variety of access/distribution options
|
||||
Data in the repository shall have sufficient technical metadata to
|
||||
FREQ.24 UC2 assure functionality (e.g. viewing and display) to ensure accessibility Must City Data, Platform
|
||||
and reusability.
|
||||
Allows publishers to display and perform manual/visual quality control Business Needs, City
|
||||
FREQ.25 UC2 Could
|
||||
assurance via a user-friendly GUI Data, Platform
|
||||
Business Needs, City
|
||||
FREQ.26 UC2 Any errors shall prompt a request for resubmission of data Should
|
||||
Data, Platform
|
||||
|
||||
FREQ.27 UC3 Enable data providers to manage their resources Must Business Needs
|
||||
|
||||
A minimal set of identifying information/metadata concerning data Business Needs,
|
||||
FREQ.28 UC3 Must
|
||||
publication submission must be recorded Platform
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 20
|
||||
|
||||
|
||||
Stores and tracks versions of data. Links /connections between
|
||||
FREQ.29 UC3 Must City Data, Platform
|
||||
versions are created and maintained
|
||||
Give service and data providers access to anonymized data of the Business Needs
|
||||
FREQ.30 UC3 Should
|
||||
subscribers of their data or services
|
||||
City Data, Platform,
|
||||
FREQ.31 UC3 Enable data providers to maintain and repair data and metadata Should
|
||||
Business Needs
|
||||
Tracks data publication agreements between Data and Platform Business Needs,
|
||||
FREQ.32 UC3 Must
|
||||
Providers Platform
|
||||
Store terms of agreements, and use them to monitor/review/process
|
||||
FREQ.33 UC3 Must City Data, Platform
|
||||
data submissions.
|
||||
Able to add and edit terms of agreement, based on access of level of Business Needs,
|
||||
FREQ.34 UC3 Must
|
||||
user. Platform
|
||||
|
||||
FREQ.35 UC3 Submission volumes and schedules are managed and monitored Must City Data, Platform
|
||||
|
||||
|
||||
|
||||
|
||||
2.4 SUB-GOAL 2: City data is managed in a safe and intelligent manner
|
||||
Rationale: The urban platform enables users to publish, consume and commercialise data, as well
|
||||
as deploy and manage services all in a secure and privacy protected manner.
|
||||
|
||||
Achieve [City data is exploited
|
||||
to its full effect]
|
||||
Goal
|
||||
co-ena bles
|
||||
|
||||
|
||||
|
||||
|
||||
Maintain [Resources are managed in a
|
||||
safe and intelligent manner]
|
||||
S ub-Goal 1
|
||||
|
||||
|
||||
|
||||
|
||||
Achieve [Store Data] Achieve [Transmit Data] Achieve [Manage Infrastructure]
|
||||
UC4 UC5 UC6
|
||||
|
||||
|
||||
Figure 6. Sub-Goal 2 “City data is managed in a safe and intelligent manner” refinement.
|
||||
|
||||
|
||||
|
||||
Drivers: Ensure data is secured and the identity of users are preserved
|
||||
Actions: For Sub-Goal 2 to be maintained in the long-run it requires the efficient realisation of use
|
||||
cases: “Store City Data” and “Retrieve and Transmit City Data”, as shown in Figure 6.
|
||||
|
||||
|
||||
2.4.1 Use Case: Store City Data
|
||||
ID: UC4
|
||||
Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner
|
||||
Pre-condition: Data is successfully published in the platform
|
||||
Actors: Urban Platform
|
||||
Rationale: When data is successfully submitted (either via APIs or manual upload), it is
|
||||
processed/prepared for storage into the platform’s database. This procedure will include the
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 21
|
||||
|
||||
|
||||
generation of unique identifiers to the database, enrichment with ontologies (when applicable),
|
||||
encrypted (when applicable), signed with digital certificates (when applicable) to ensure that the
|
||||
data conforms to the platform data formatting, standards, security and regulation. Data may be
|
||||
converted to accepted formats, as needed (e.g. graph model). A primary goal of the conversion of
|
||||
content for the platform is the preservation of the content. Priority will be given to preserving the
|
||||
data accordingly to the policies defined in section (2.6.2). Access-control levels and license models
|
||||
are associated to data which is subject to restrictions relating to access and conditions of use.
|
||||
Refines into requirements: FREQ 28 – FREQ 39.
|
||||
|
||||
Use Case Basic Interactions and Responses
|
||||
1. Platform mechanisms converts submitted data into a standard format
|
||||
2. Security enforcement (e.g. anonymisation, cryptography) is placed on
|
||||
sensitive information.
|
||||
UC4. Store City 3. Platform associates with datasets the access-control definitions set by
|
||||
Data owner of resources.
|
||||
4. Data is enriched with semantics and is associated with other datasets
|
||||
5. Platform stores data in a scalable and secure database.
|
||||
6. End of data storage.
|
||||
|
||||
|
||||
2.4.2 Use Case: Transmit Data
|
||||
ID: UC5
|
||||
Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner
|
||||
Pre-condition: Data is successfully published in the platform
|
||||
Actors: Urban Platform
|
||||
Rationale: The platform accepts data retrieval request, validates user’s rights to access the data,
|
||||
retrieves city data from data storage, and moves a copy of the data to the relevant platform
|
||||
component for further processing. If special processing is required, the retrieval function accesses
|
||||
data in staging storage and applies the requested processes. The types of operations, which may
|
||||
be carried out, include sub-sampling in temporal or spatial dimensions, conversions between
|
||||
different data types or output formats, and other specialized processing (e.g., data visualisation).
|
||||
Once it is finalised data will be sent to the appropriate delivery channels (e.g. API’s, GUI). It also
|
||||
encompasses function to verify corruption during any internal data transfer. This function requires
|
||||
that all hardware and software within the platform provide notification of potential errors and that
|
||||
these errors are routed to standard error logs that are checked by the Platform Provider.
|
||||
Refines into requirements: FREQ 40 to 54.
|
||||
2.4.3 Use Case: Manage Infrastructure
|
||||
ID: UC6
|
||||
Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner
|
||||
Pre-condition: The platform is available
|
||||
Actors: Urban Platform
|
||||
Rationale: Manage infrastructure provides the services and functions for the overall operation of
|
||||
the urban platform. Administration functions include monitoring quality of service agreements,
|
||||
auditing data publication to ensure that they meet archive standards, and maintaining configuration
|
||||
management of system hardware and software. In overall, it provides system engineering
|
||||
functions to monitor and improve platform operations, and to inventory, report on, and
|
||||
migrate/update the contents of the platforms’ databases.
|
||||
Refines into requirements: FREQ 55 to 62.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 22
|
||||
|
||||
|
||||
|
||||
Use Case Basic Interactions and Responses
|
||||
1. Platform keeps monitoring services at run-time to ensure operation
|
||||
and integrity of city data
|
||||
o If system failure, the platform activates mechanisms for recovery
|
||||
UC6. Manage based on pre-defined rules
|
||||
Infrastructure o Platform logs issue and issue alert messages to platform providers
|
||||
2. Platform logs operation capabilities (e.g. performance, mean of time
|
||||
failure, issues, etc.)
|
||||
|
||||
|
||||
2.4.4 Functional Requirements
|
||||
|
||||
Req. ID UC. ID Description Priority Domain
|
||||
A minimal set of identifying information/metadata concerning data Business Needs,
|
||||
FREQ.28 UC4 Must
|
||||
publication submission must be recorded. Platform
|
||||
Stores and tracks versions of data. Links /connections between
|
||||
FREQ.29 UC4 Must City Data, Platform
|
||||
versions are created and maintained.
|
||||
|
||||
FREQ.30 UC4 Converts data to accepted file formats Must City Data, Platform
|
||||
|
||||
Keep sensitive information secured and accessible only to authorized
|
||||
FREQ.31 UC4 Must City Data, Platform
|
||||
users
|
||||
|
||||
FREQ.32 UC4 Keep user’s personal information protected Should City Data, Platform
|
||||
|
||||
|
||||
FREQ.33 UC4 Keep city data and meta-data secured Must Platform Needs
|
||||
|
||||
|
||||
FREQ.34 UC4 Enable privacy preserving mechanisms associated to data Must Platform Needs
|
||||
|
||||
|
||||
FREQ.35 UC4 Model data in accordance with defined standards Must City Data, Platform
|
||||
|
||||
|
||||
FREQ.36 UC4 Support the use of ontologies and semantic modelling of city data Could City Data
|
||||
|
||||
|
||||
FREQ.37 UC4 Support database-level provenance annotation Should City Data
|
||||
|
||||
|
||||
FREQ.38 UC4 Support data-level provenance annotation Should City Data
|
||||
|
||||
|
||||
FREQ.39 UC4 Enable data to be encrypted Should Platform Needs
|
||||
|
||||
System must have the ability to search and display metadata, preferably
|
||||
City Data, Platform
|
||||
FREQ.40 UC5 in a user-conformable, human readable display as well as in its native Must
|
||||
Needs
|
||||
format for machine harvesting and manipulation.
|
||||
Controls access to data in the repository based on multiple permission
|
||||
City Data, Platform
|
||||
FREQ.41 UC5 levels. These permission levels determine the create/edit/read/delete Should
|
||||
Needs
|
||||
privileges granted users.
|
||||
Access rights and conditions of use will be held for each data and its City Data, Platform
|
||||
FREQ.42 UC5 Should
|
||||
related metadata. Needs
|
||||
Access rights and conditions can be inherited from a parent data to any City Data, Platform
|
||||
FREQ.43 UC5 Could
|
||||
data designated as a child data (derived information). Needs
|
||||
Access rights and conditions of use will be machine readable and City Data, Platform
|
||||
FREQ.44 UC5 Should
|
||||
actionable. Needs
|
||||
Access mechanisms must be sufficiently granular to allow the Business Needs, City
|
||||
FREQ.45 UC5 identification of individual users, in order to maintain audit logs of actions Should
|
||||
Data, Platform
|
||||
performed by users.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 23
|
||||
|
||||
|
||||
Maintains the integrity of the database which contains both metadata
|
||||
FREQ.46 UC5 and system information. Must Platform Needs
|
||||
|
||||
Provides internal validation such as referential integrity of the contents of City Data, Platform
|
||||
FREQ.47 UC5 the database. Must
|
||||
Needs
|
||||
Creates and maintains schema definitions required to support data
|
||||
FREQ.48 UC5 management functions. Must Platform Needs
|
||||
|
||||
Monitors and ensures that data and metadata are not corrupted during
|
||||
FREQ.49 UC5 transfers. Must Platform Needs
|
||||
|
||||
Provides statistically acceptable assurance that no components of the
|
||||
FREQ.50 UC5 data are corrupted during any internal data transfer. Must Platform Needs
|
||||
|
||||
Performs routine and special data integrity checking for each dataset
|
||||
FREQ.51 UC5 and generates error reports. Must Platform Needs
|
||||
|
||||
Provides disaster recovery capabilities including data backup, off-site
|
||||
FREQ.52 UC5 data storage, data recovery, etc. Must Platform Needs
|
||||
|
||||
Refresh/replace data without service interruption, and update
|
||||
FREQ.53 UC5 corresponding metadata as appropriate. Must Platform Needs
|
||||
|
||||
Ensure that any associated unique identifiers of the updated data are not
|
||||
FREQ.54 UC5 altered. Must Platform Needs
|
||||
|
||||
Audits submissions to ensure that they meet archive/repository
|
||||
FREQ.55 UC6 standards. Must Platform Needs
|
||||
|
||||
Maintains configuration management of the system hardware and
|
||||
FREQ.56 UC6 software. Must Platform Needs
|
||||
|
||||
Has capability to inventory, report on and migrate the contents of the
|
||||
FREQ.57 UC6 repository. Must Platform Needs
|
||||
|
||||
|
||||
FREQ.58 UC6 Ensures data integrity for version upgrades and format migration. Must Platform Needs
|
||||
|
||||
|
||||
FREQ.59 UC6 Monitors functionality of the entire repository. Must Platform Needs
|
||||
|
||||
|
||||
FREQ.60 UC6 Maintains integrity of system configuration. Must Platform Needs
|
||||
|
||||
Audits system operations, performance and usage.
|
||||
FREQ.61 UC6 Must Platform Needs
|
||||
|
||||
Provides platform performance information and database holdings
|
||||
FREQ.62 UC6 inventory reports Must Platform Needs
|
||||
|
||||
|
||||
|
||||
|
||||
2.5 SUB-GOAL 3: City data is orchestrated in a marketplace
|
||||
Rationale: The urban platform enables users to consume and publish data in a secure and privacy
|
||||
protected manner.
|
||||
Drivers: Ensure data is secured and the identity of users are preserved
|
||||
Actions: For Sub-Goal 3 to be maintained in the long-run it requires the efficient realisation of use
|
||||
cases: “Commercialise City Data” and “Commercialise Data Services”, as shown in Figure 7.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 24
|
||||
|
||||
|
||||
|
||||
Achieve [City data is exploited
|
||||
to its full effect]
|
||||
Goal
|
||||
|
||||
|
||||
|
||||
|
||||
co-ena bles
|
||||
Maintain [City data is orchestrated in
|
||||
a marketplace]
|
||||
S ub-Goal 3
|
||||
|
||||
|
||||
|
||||
|
||||
Achieve [Commercialise Data
|
||||
Achieve [Commercialise City Data]
|
||||
Services]
|
||||
UC7 UC8
|
||||
|
||||
|
||||
|
||||
|
||||
Figure 7. Sub-Goal 3 “City data is orchestrated in a marketplace” refinement.
|
||||
|
||||
|
||||
|
||||
2.5.1 Use Case: Commercialise City Data
|
||||
ID: UC7
|
||||
Refines: SUB-GOAL 3: City data is orchestrated in a marketplace
|
||||
Pre-condition: Data is successfully published in the platform, both publisher and consumers of
|
||||
city data are authenticated in the platform, and there are billing capabilities available.
|
||||
Actors: Data Publishers, Data Consumers
|
||||
Rationale: The providers of city data can commercialise city data based on the policies and
|
||||
financial models defined in the platform. After publishing their data, publishers can define which
|
||||
data can be available as open data and which data should be available with the payment of a
|
||||
subscription fee. Once publishers define which data is to be commercially exploited, the platform
|
||||
will associate the data with their respective financial models and let it ready for subscription. City
|
||||
data consumer chooses which data to purchase and is redirected to a billing interface where the
|
||||
subscription payment is taken. The platform must provide an update response indicating the status
|
||||
of the payment. If successful, data is ready available to be consumed by humans and machines,
|
||||
otherwise the user can re-try the payment or cancel transaction.
|
||||
Commercialise city data also involves the function of managing commercial data. It provides
|
||||
services and functions for updating, maintaining and accessing both data and its respective
|
||||
commercial transactions. Furthermore, it enables data providers to track the usage of commercial
|
||||
data by users. Ideally the owners of the data should be the only authorised user to manage
|
||||
resources, and other authorised users can track the usage of the data in the platform. Data usage
|
||||
tracking includes performing queries on the data management data to generate result sets, and
|
||||
producing reports from these result sets. Data consumers are provided with functions which enable
|
||||
them to manage their subscriptions and financial transactions on the platform. These functions
|
||||
include updating, maintaining and accessing financial transactions. For all these functions and
|
||||
services, the platform must provide a database update response indicating the status of the
|
||||
update, avoid update errors to be propagated in the platform, and should keep an audit trail of all
|
||||
actions to enable rollback.
|
||||
Specialised Use Cases: The Use Case Commercialise City Data is distinguished into four
|
||||
specialised Use Cases: “Commercialise Data (UC7.1)”, “Consume Commercial City Data (UC7.2)”,
|
||||
“Manage Commercial Data (UC7.3)” and “Manage Data Subscription (UC7.4)”.
|
||||
Refines into requirements: FREQ 63 to 68.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 25
|
||||
|
||||
|
||||
|
||||
|
||||
Use Case Basic Interactions and Responses
|
||||
1. User selects the data to be commercialised
|
||||
2. User selects the commercial model for data consumption
|
||||
UC7.1. 3. Platform validates selection
|
||||
Commercialise 4. Platform associates data to subscription model
|
||||
Data 5. Platform releases data for commercial exploitation in the marketplace
|
||||
6. End of data commercialisation set up.
|
||||
1. User selects the data to be subscribed to
|
||||
2. User request subscription to data
|
||||
UC7.2. 3. Platform validates selection
|
||||
Consume 4. Platform redirects user to billing system
|
||||
Proprietary 5. Billing system deals with user request
|
||||
Data o If successful, user is redirected to a GUI where data is ready to use
|
||||
o If unsuccessful, user can try payment again or cancel request
|
||||
2. End of data subscription.
|
||||
1. Platform provides user with an interface for commercial data
|
||||
management
|
||||
2. User chooses to edit or delete commercial data
|
||||
o If edit, user revise commercial models, licenses, access-control,
|
||||
semantics;
|
||||
o If delete, user selects dataset(s) to be removed (following policies
|
||||
UC7.3. Manage defined in the platform for data removal)
|
||||
commercial 3. User confirms action
|
||||
Data 4. Platform promptly process user’s request
|
||||
5. Platform confirms execution of request
|
||||
o If valid request, platform acknowledges request has been
|
||||
processed successfully
|
||||
o If non-valid request, platform returns to step 1.
|
||||
6. End of resources management
|
||||
1. Platform provides user with an interface for data subscription
|
||||
management
|
||||
2. User chooses to edit or cancel data subscription
|
||||
o If edit, user revise payment and subscription timeframe;
|
||||
o If cancel, user selects dataset(s) to have subscription cancelled
|
||||
UC7.4. Manage (following policies defined in the platform for data subscription)
|
||||
data 3. User confirms action
|
||||
subscription 4. Platform promptly process user’s request
|
||||
5. Platform confirms execution of request
|
||||
o If valid request, platform updates acknowledges request has been
|
||||
processed successfully
|
||||
o If non-valid request, platform returns to step 1.
|
||||
6. End of data subscription management
|
||||
|
||||
|
||||
|
||||
2.5.2 Use Case: Commercialise Data Services
|
||||
ID: UC8
|
||||
Refines: SUB-GOAL 3: City data is orchestrated in a marketplace
|
||||
Pre-condition: Data is successfully published in the platform, both publisher and consumers of
|
||||
city data are authenticated in the platform, and there are billing capabilities available.
|
||||
Actors: Data Service Providers, Data Publishers and Consumers
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 26
|
||||
|
||||
|
||||
Rationale: After deploying data services in the platform, the providers of data services can choose
|
||||
to commercialise services based on the policies and financial models defined in the platform. Once
|
||||
data service providers define which service(s) is (are) to be commercially exploited, the platform
|
||||
will associate the services with their respective financial models and let available in the platform
|
||||
applications module ready for use. User (either publisher or consumer) chooses which data
|
||||
services to use, and in case a charged service is selected the platform redirects the user to a
|
||||
billing interface where the payment is taken. The platform must provide an update response
|
||||
indicating the status of the payment. If successful, service is ready available to be used, otherwise
|
||||
the user can re-try the payment or cancel transaction. Note that data service owners should be
|
||||
able to waive the payment of tariff to certain users’ categories.
|
||||
Data services providers are offered with functions to manage their commercial services. The
|
||||
platform provides functions for updating, maintaining and accessing both service and its respective
|
||||
commercial transactions. Furthermore, it enables data services providers to track the usage of
|
||||
services by users. Services usage tracking includes performing queries on the platform to
|
||||
generate result sets, and producing reports from these result sets. The consumers of data services
|
||||
are provided with functions which enable them to manage their subscriptions and financial
|
||||
transactions. These functions include updating, maintaining and accessing financial transactions.
|
||||
For all these functions and services, the platform must provide a database update response
|
||||
indicating the status of the update, avoid update errors to be propagated in the platform, and
|
||||
should keep an audit trail of all actions to enable rollback.
|
||||
Specialised Use Cases: The Use Case Commercialise Data Services is distinguished into four
|
||||
specialised Use Cases: “Commercialise Data Services (UC8.1)”, “Consume Data Services
|
||||
(UC8.2)”, “Manage Commercial Services (UC8.3)” and “Manage Services Subscription (UC8.4)”.
|
||||
Refines into requirements: FREQ 68 to 73.
|
||||
|
||||
Use Case Basic Interactions and Responses
|
||||
1. User selects the data services to be commercialised
|
||||
2. User selects the commercial model for data service usage based on the
|
||||
category of users
|
||||
UC8.1. 3. Platform validates selection
|
||||
Commercialise 4. Platform associates data services to subscription model
|
||||
Data Services 5. Platform enables data service to be commercially exploited in the
|
||||
marketplace
|
||||
6. End of services commercialisation set up.
|
||||
1. User selects the data service to be subscribed to
|
||||
2. User request subscription to service
|
||||
3. Platform validates selection
|
||||
UC8.2. 4. Platform validates user’s category its respective commercial models
|
||||
Consume Data 4. If applicable, platform redirects user to billing system
|
||||
Services 5. Billing system deals with user’s request
|
||||
o If successful, user is redirected to a GUI where data is ready to use
|
||||
o If unsuccessful, user can try payment again or cancel request
|
||||
6. End of data subscription.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 27
|
||||
|
||||
|
||||
1. Platform provides user with an interface for commercial data
|
||||
management
|
||||
2. User chooses to edit or delete commercial data
|
||||
o If edit, user revise commercial models, licenses, access-control,
|
||||
semantics;
|
||||
o If delete, user selects dataset(s) to be removed (following policies
|
||||
UC8.3. Manage defined in the platform for data removal)
|
||||
commercial 3. User confirms action
|
||||
services 4. Platform promptly process user’s request
|
||||
5. Platform confirms execution of request
|
||||
o If valid request, platform acknowledges request has been
|
||||
processed successfully
|
||||
o If non-valid request, platform returns to step 1.
|
||||
6. End of resources management
|
||||
1. Platform provides user with an interface for data subscription
|
||||
management
|
||||
2. User chooses to edit or cancel data subscription
|
||||
o If edit, user revise payment and subscription timeframe;
|
||||
o If cancel, user selects dataset(s) to have subscription cancelled
|
||||
UC8.4. Manage (following policies defined in the platform for data subscription)
|
||||
services 3. User confirms action
|
||||
subscription 4. Platform promptly process user’s request
|
||||
5. Platform confirms execution of request
|
||||
o If valid request, platform updates acknowledges request has been
|
||||
processed successfully
|
||||
o If non-valid request, platform returns to step 1.
|
||||
6. End of data subscription management
|
||||
|
||||
|
||||
2.5.3 Functional Requirements
|
||||
|
||||
Req. ID UC. ID Description Priority Domain
|
||||
City Data, Platform,
|
||||
FREQ.63 UC7 Support the commercialization of city data Should Business Needs
|
||||
City Data, Platform,
|
||||
FREQ.64 UC7 Enable users to subscribe to city data through the payment of a tariff Should Business Needs
|
||||
City Data, Platform,
|
||||
FREQ.65 UC7 Enable users to manage their data subscriptions Should Business
|
||||
Provide platform providers mechanisms to define the terms and
|
||||
FREQ.66 UC7 Must Platform Needs
|
||||
conditions for platform data usage
|
||||
Enable data providers to manage the subscription models of their City Data, Platform,
|
||||
FREQ.67 UC7 Should
|
||||
data Business
|
||||
UC7/ City Data, Platform,
|
||||
FREQ.68 Utilise secure and reliable billing and payment management systems Must
|
||||
UC8 Business
|
||||
City Data, Platform,
|
||||
FREQ.69 UC8 Support the commercialization of data services Should Business Needs
|
||||
Enable data providers to manage the commercial models of their City Data, Platform,
|
||||
FREQ.70 UC8 Should
|
||||
services Business Needs
|
||||
Provide service providers mechanisms to define the terms and City Data, Platform,
|
||||
FREQ.71 UC8 Should
|
||||
conditions of platform services Business Needs
|
||||
Allow users to pay a tariff for using certain advanced services (e.g. City Data, Platform,
|
||||
FREQ.72 UC8 Should
|
||||
Data manipulation, enrichment) Business Needs
|
||||
City Data, Platform,
|
||||
FREQ.73 UC8 Enable users to manage their data services subscriptions Should Business Needs
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 28
|
||||
|
||||
|
||||
2.6 SUB-GOAL 4: City data is offered in an accessible manner
|
||||
Rationale: The urban platform provides city data in both human and machine (e.g. sensors,
|
||||
actuators, systems) readable and understandable formats.
|
||||
Drivers: Ensure data understandability and machine-to-machine data transaction.
|
||||
Actions: For Sub-Goal 3 to be maintained in the long-run it requires the efficient realisation of use
|
||||
cases: “Register Consumer”, “Discover City Data”, and “Consume City Data” as shown in Figure 8.
|
||||
|
||||
Achieve [City data is exploited
|
||||
to its full effect]
|
||||
Goal
|
||||
|
||||
|
||||
|
||||
|
||||
co-ena bles
|
||||
Maintain [City data is offered in an
|
||||
accessible manner]
|
||||
S ub-Goal 1
|
||||
|
||||
|
||||
|
||||
|
||||
Achieve [Register Consumer] Achieve [Discover City Data] Achieve [Consume City Data]
|
||||
UC9 UC10 UC11
|
||||
|
||||
|
||||
|
||||
|
||||
Figure 8. Sub-Goal 4 “City data is offered in an accessible manner” refinement.
|
||||
|
||||
|
||||
|
||||
2.6.1 Use Case: Register Data Consumer
|
||||
ID: UC9
|
||||
Refines: SUB-GOAL 4: City data is offered in an accessible manner
|
||||
Pre-condition: User is not logged in the platform
|
||||
Actors: Data Consumers
|
||||
Rationale: Data Consumers can register in the platform and request approval to consume city data
|
||||
via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to
|
||||
confirm their registration. Users must accept the terms and conditions of platform usage and define
|
||||
how their personal data can be used by the Platform Owner. Users can manage and alter their
|
||||
registration information at any time they want to.
|
||||
Refines into requirements: FREQ 64.
|
||||
|
||||
Use Case Basic Stimulus and Responses
|
||||
1. The platform prompts the user for a username and password or register
|
||||
new account.
|
||||
2. The user selects registration option.
|
||||
3. The platform prompts user for data consumer registration information
|
||||
UC1. Register 4. The user enters in their information.
|
||||
Consumer 5. Platform verifies information and creates account.
|
||||
o If non-valid information, platform shows error message and returns to
|
||||
step 1.
|
||||
6. Platform acknowledges registration has been successful
|
||||
7. End of registration
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 29
|
||||
|
||||
|
||||
2.6.2 Use Case: Discover City Data
|
||||
ID: UC10
|
||||
Refines: SUB-GOAL 4: City data is offered in an accessible manner
|
||||
Pre-condition: User has access to either platform GUI or API
|
||||
Actors: Data Consumers
|
||||
Rationale: Data Consumers can register in the platform and request approval to consume city data
|
||||
via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to
|
||||
confirm their registration.
|
||||
Refines into requirements: FREQ 64.
|
||||
|
||||
Use Case Basic Stimulus and Responses
|
||||
1. Users access specialised data query end-points (e.g. SPARQL)
|
||||
2. Users provides information for pre-defined parameters for search
|
||||
3. Users request data search
|
||||
4. Platform quickly process users request for data
|
||||
604
files/core_requirements_part5.txt
Normal file
604
files/core_requirements_part5.txt
Normal file
@@ -0,0 +1,604 @@
|
||||
Consume City o If authentication is successful, users are provided with requested
|
||||
Data via APIs data streams
|
||||
3. Users are provided with requested data via APIs
|
||||
|
||||
|
||||
|
||||
2.6.4 Functional Requirements
|
||||
|
||||
Req. ID UC. ID Description Priority Domain
|
||||
Allow users to register to use services and consume proprietary city Societal Needs,
|
||||
FREQ.74 UC9 Should
|
||||
data and open data (optional) Platform
|
||||
Keep sensitive information secured and accessible only to Societal Needs,
|
||||
FREQ.75 UC9 Should
|
||||
authorized users Platform
|
||||
Societal Needs,
|
||||
FREQ.76 UC9 Provide authentication mechanisms for users Must Platform
|
||||
Societal Needs,
|
||||
FREQ.77 UC9 Keep user’s personal information protected Must Platform
|
||||
Allow users to control which data they are willing to provide and how Societal Needs,
|
||||
FREQ.78 UC9 Must
|
||||
their data should be used Platform
|
||||
Societal Needs,
|
||||
FREQ.79 UC10 Allow users to format data in any supported data formats Must
|
||||
Platform
|
||||
The query request may require data to be sourced from different Societal Needs,
|
||||
FREQ.80 UC10 Must
|
||||
storage locations Platform
|
||||
Allows query requests against all metadata used to manage the Societal Needs,
|
||||
FREQ.81 UC10 repository. Should Platform
|
||||
Provide users information about the legal aspects of the data Societal Needs,
|
||||
FREQ.82 UC11 Must
|
||||
(license, ownership) City Data
|
||||
Societal Needs,
|
||||
FREQ.83 UC11 Keeps an audit trail of all actions. Must City Data
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 31
|
||||
|
||||
|
||||
2.7 SUB-GOAL 5: User’s experience is enhanced by the provision of
|
||||
value-added services
|
||||
Rationale: The urban platform enables users to consume and publish data in a secure and privacy
|
||||
protected manner
|
||||
Drivers: Ensure data is secured and the identity of users are preserved
|
||||
Actions: For Sub-Goal 5 to be maintained in the long-run it requires the efficient realisation of use
|
||||
cases: “Deploy Data Services”, “Manage Services”, and “Utilise Data Services” as shown in Figure
|
||||
8.
|
||||
|
||||
Achieve [City data is exploited
|
||||
to its full effect]
|
||||
Goal
|
||||
|
||||
|
||||
|
||||
|
||||
co-ena bles
|
||||
Maintain [User s experience is enhanced by
|
||||
the provision of value-added services]
|
||||
S ub-Goal 1
|
||||
|
||||
|
||||
|
||||
|
||||
Achieve [Deploy Data Services] Achieve [Manage Services] Achieve [Utilise Data Services]
|
||||
UC12 UC13 UC14
|
||||
|
||||
|
||||
|
||||
|
||||
Figure 9. Sub-Goal 5 “User’s experience is enhanced by the provision of value-added services” refinement
|
||||
|
||||
|
||||
|
||||
2.7.1 Deploy Data Services
|
||||
ID: UC12
|
||||
Refines: SUB-GOAL 5: User’s experience is enhanced by the provision of value-added services
|
||||
Pre-condition: Data Services Providers are provided with credentials, are authorised to deploy
|
||||
their services in the platform, and have access to technical specifications of the platform interfaces.
|
||||
Actors: Data Services Providers
|
||||
Rationale: Data Services can register in the platform and request approval to deploy services in
|
||||
the platform. They provide valid registration details (to be defined) and wait for registration
|
||||
confirmation. Platform Providers may authorise or not data services providers to offer both open
|
||||
and proprietary services in the platform. Data services providers must formally agree with service
|
||||
deployment agreement with the Urban Platform. This agreement defines terms of the content,
|
||||
policies, regulations, license agreement. The Urban Platform will proactively work with Service
|
||||
Providers to agree on the technical specifications of interfaces and platform openness level.
|
||||
Agreements between Platform and Data Service Providers may be renegotiated on a periodic or
|
||||
ad-hoc basis.
|
||||
Refines into requirements: FREQ.1 to FREQ.5.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 32
|
||||
|
||||
|
||||
|
||||
Use Case Basic Stimulus and Responses
|
||||
1. User authenticates in the platform
|
||||
2. User is provided access to platform interfaces for service deployment
|
||||
3. User deploy services
|
||||
4. Platform checks compatibility and any technical issues arising from the
|
||||
UC12. Deploy new service
|
||||
Services 5. If approved the service is ready to be used and managed in the
|
||||
platform
|
||||
o If deployment is not approved, platform shows error message and
|
||||
returns to step 1.
|
||||
6. End of deployment
|
||||
|
||||
|
||||
|
||||
2.7.2 Use Case: Manage Services
|
||||
ID: UC13
|
||||
Refines: GOAL 5: User’s experience is enhanced by the provision of value-added services
|
||||
Pre-condition: User successfully authenticates in the platform
|
||||
Actors: Data Services Providers
|
||||
Rationale: Manage services provides the functions for updating, maintaining and accessing
|
||||
services as well as tracking their usage by users. Ideally the owners of the services should be the
|
||||
only authorised user to manage them, and other authorised users can track the usage of the
|
||||
services in the platform. In case of updates the platform must log in the database update details
|
||||
and send to service providers a response indicating the status of the update. The platform should
|
||||
also ensure update errors are not propagated nor affect the health of other services provided in the
|
||||
platform, and should keep an audit trail of all actions to enable rollback. Data services usage
|
||||
tracking includes performing queries on the data management data to generate result sets, and
|
||||
producing reports from these result sets. All user’s information provided to service providers must
|
||||
follow regulations of data protection and the user’s defined rules for their data use.
|
||||
Specialised Use Cases: The Use Case Manage Services is distinguished into two specialised
|
||||
Use Cases: “User manages services (UC13.1)” and “User tracks service usage (UC13.2)”.
|
||||
Subordinated Use Cases: “Transmit Data (UC5)”
|
||||
Refines into requirements: FREQ.27 to FREQ.25.
|
||||
|
||||
|
||||
Specialised Use Cases Basic Interactions and Responses
|
||||
1. Platform provides user with an interface for services management
|
||||
2. User chooses to edit or delete services
|
||||
3. If edit, user revise service information (access-control,
|
||||
commercial models, parameters) and deployment;
|
||||
If delete, user selects services to be removed / disabled
|
||||
UC13.1. User manages 4. User confirms action
|
||||
services 5. Platform quickly process user’s request
|
||||
6. Platform confirms execution of request
|
||||
o If valid request, platform acknowledges request has been
|
||||
processed successfully
|
||||
o If non-valid request, platform returns to step 1.
|
||||
7. End of services management
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 33
|
||||
|
||||
|
||||
1. Platform provides user with an interface for services management
|
||||
2. User chooses to visualise usage information of a service
|
||||
3. Platform quickly process user’s request for data usage
|
||||
UC13.2. User tracks information
|
||||
services usage 4. Platform provides user with statistical information about services
|
||||
usage and data users anonymised information
|
||||
5. End of data services tracking.
|
||||
|
||||
|
||||
|
||||
2.7.3 Use Case: Utilise Data Services
|
||||
ID: UC2
|
||||
Refines: SUB-GOAL 1 - City data is collected in an intelligent manner
|
||||
Pre-condition: User is authenticated in the platform
|
||||
Actors: City data publisher
|
||||
Rationale: Data Consumers can register in the platform and request approval to consume city data via GUI
|
||||
or APIs. They provide valid registration details (to be defined) and wait for platform to confirm their
|
||||
registration.
|
||||
Subordinated Use Cases: “Commercialise Data Services (UC8)”
|
||||
Refines into requirements: FREQ 6 to FREQ.26.
|
||||
|
||||
|
||||
Specialised Use Cases Basic Interactions and Responses
|
||||
1. Users / Machines select data service to be utilised
|
||||
2. Users / Machines are redirected to authentication mechanism in
|
||||
UC2.1. User utilises data case of registration is needed for the particular service
|
||||
services o If authentication is successful, users are provided with
|
||||
requested data streams
|
||||
3. Users are provided with requested service either via API or GUI
|
||||
|
||||
|
||||
2.7.4 Functional Requirements
|
||||
|
||||
Req. ID UC. ID Description Priority Domain
|
||||
Provide stable and well-defined interfaces to ensure interoperability
|
||||
Societal Needs,
|
||||
FREQ.84 UC12 between the platform, services and the applications provided by Should Platform
|
||||
services providers
|
||||
Ensure the interfaces of the architecture are open to reduce entry Societal Needs,
|
||||
FREQ.85 UC12 Should
|
||||
barriers and integration issues Platform
|
||||
Provide multi-purposed and network intelligent interfaces to Societal Needs,
|
||||
FREQ.86 UC12 Must
|
||||
providers and consumers of services Platform
|
||||
Provide service providers mechanisms to define the terms and Societal Needs,
|
||||
FREQ.87 UC12 Must
|
||||
conditions of platform services deployment Platform
|
||||
Business Needs,
|
||||
FREQ.88 UC13 Provide statistical information of user’s feedback on service usage Must Platform
|
||||
Allow users to use services to manipulate city data (e.g. Create Societal Needs,
|
||||
FREQ.89 UC10 Must
|
||||
mash ups, integrate) Platform
|
||||
Allow users to provide feedback on usability, and quality of data Societal Needs,
|
||||
FREQ.90 UC14 Must
|
||||
and services provided by the platform Platform
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 34
|
||||
|
||||
|
||||
|
||||
3. Non-functional Requirements
|
||||
3.1 Run-time Quality Requirements
|
||||
3.1.1 Scalability Requirements
|
||||
|
||||
Rationale: The ability of the system to execute its task within its expected performance profile and
|
||||
to handle on-demand increased processing volumes of data and service requests
|
||||
Drivers: Provide ready access to all data that underpins decision making processes in smart cities,
|
||||
and accommodate users and data usage patterns
|
||||
Refines into requirements: NFREQ.1, NFREQ.2, NFREQ.3, NFREQ.4
|
||||
Relevance: Urban platforms have ambitious performance requirements. Such platforms must cope
|
||||
with user’s demand for data and services, capture real-time data that will be catalysed by a myriad
|
||||
of sensors. The demand for urban platform is very likely to significantly increase over time. It is
|
||||
very difficult to have clear performance characteristics due to the ubiquity, heterogeneity high
|
||||
connectivity of devices and end users.
|
||||
|
||||
SCALABILITY MEASURES
|
||||
Actions Capture the performance requirements
|
||||
Create service level agreements
|
||||
Predict scalability using software simulation
|
||||
Analyse the performance of the platform overtime
|
||||
Conduct practical testing
|
||||
Strategy Prioritize service and data requests
|
||||
Distribute processing over time
|
||||
Scale up or scale out as necessary
|
||||
Reuse resources and results Partition and parallelize
|
||||
Constantly monitor Quality of Service at runtime
|
||||
|
||||
|
||||
|
||||
3.1.2 Availability and Reliability Requirements
|
||||
|
||||
Rationale: The ability of the system to be fully or partly operational as and when required and to
|
||||
effectively handle failures that could affect system availability
|
||||
Drivers: Build a reliable foundation for “on demand” exploitation of data
|
||||
Refines into requirements: NFREQ.5, NFREQ.6, NFREQ.7, NFREQ.8
|
||||
Relevance: Any system that has complex or extended availability requirements, complex recovery
|
||||
processes, or a high profile (e.g., is visible to the public)
|
||||
|
||||
AVAILABILITY AND RELIABILITY MEASURES
|
||||
Actions Capture the availability requirements Produce the availability schedule
|
||||
Estimate platform availability Estimate functional availability Assess against the
|
||||
requirements Rework the architecture
|
||||
Strategy Adopt fault-tolerant hardware
|
||||
Use reliable infrastructure database
|
||||
Log transactions
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 35
|
||||
|
||||
|
||||
Develop adaptive software to cope and recover from faults
|
||||
Design and test for failure
|
||||
Deploy load balancing
|
||||
Identify suitable backup and disaster recovery solution
|
||||
|
||||
|
||||
3.1.3 Trust Requirements
|
||||
|
||||
Rationale: A quality related to the user’s belief in the reliability, integrity and ability of the functional
|
||||
behaviour of the platform
|
||||
Drivers: Gain understanding of what influences user’s experience while interacting with services
|
||||
provided
|
||||
Requirements: NFREQ.16, NFREQ.17
|
||||
Relevance: Relevant to the systems that share and collect information that may raise public
|
||||
concern. In some cases, trust has to do with the reliability of data and their providers, whereas in
|
||||
other cases trust can be associated with the security and privacy of the technology that was
|
||||
deployed. Trust affects the reputation of the platform besides its dissemination and maturity on the
|
||||
market.
|
||||
|
||||
TRUST MEASURES
|
||||
Capture trust requirements
|
||||
Perform risk analysis so measures can be implemented
|
||||
Actions Explore the vulnerability aspects of city data
|
||||
Check whether extensibility requirements impact on trust
|
||||
Define a trust model
|
||||
Adopt trust model
|
||||
Deploy monitoring capabilities and assess its impact on scalability
|
||||
Manage the data in a way that ensures its compliance with data
|
||||
Strategy protection regulations
|
||||
Implement tampering and data misuse detection
|
||||
Make use of Cryptography when necessary
|
||||
Allow Federation of trust between platforms
|
||||
|
||||
|
||||
|
||||
3.1.4 Security Requirements
|
||||
|
||||
Rationale: Ability of the system to enforce the intended confidentiality, trust, integrity and service
|
||||
and data access policies, and to detect and recover from failure in these security mechanisms.
|
||||
Drivers: Manage the data and services in a way that ensures its integrity, and compliance with
|
||||
data protection regulations
|
||||
Relevance: Relevant to the systems that share and collect information that may raise public
|
||||
concern. Urban platforms may become a valuable target for attackers which can potentially leave
|
||||
huge swathes of information exposed. It could potentially undermine trust in the government and
|
||||
damage its reputation.
|
||||
Refines into requirements: NFREQ.9 to NFREQ.15
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 36
|
||||
|
||||
|
||||
SECURITY MEASURES
|
||||
Elicit the security requirements
|
||||
Cross-check security requirements’ impact on services offered by different
|
||||
providers
|
||||
Verify security impact on service composition
|
||||
Conduct risk analysis and impacts of security breach
|
||||
Use scalable and efficient user authentication components
|
||||
Use authentication and authorization components to secure interfacing with
|
||||
Actions external services
|
||||
Address aspects of data collection, storage and distribution
|
||||
Address aspects of service and communication security among devices
|
||||
Identify security requirements of the physical infrastructure
|
||||
Balance and prioritize scalability (performance) and security requirements
|
||||
Evaluate trade-offs between privacy considerations and preventing abuse. (While
|
||||
anonymous access guarantees privacy of users, traceability of users due to abuse
|
||||
of the service is not possible.)
|
||||
Toughen user’s functional components
|
||||
Authenticate subjects
|
||||
Define and enforce access policies
|
||||
Secure communication infrastructures
|
||||
Secure interfaces with external systems and services
|
||||
Strategy
|
||||
Secure databases
|
||||
Check data integrity for critical services
|
||||
User digital certificates and encryption where necessary
|
||||
Secure monetary transactions
|
||||
Secure user’s personal information
|
||||
|
||||
|
||||
3.1.5 Privacy Requirements
|
||||
|
||||
Rationale: Ability of the system to ensure that the collection and transmission of personal data is
|
||||
minimized and handled in accordance with user’s expectation and regulations.
|
||||
Drivers: Protect the vulnerability aspects volunteered citizen’s data
|
||||
Requirements: NFREQ.18, NFREQ.19, NFREQ.20, NFREQ.21, NFREQ.22
|
||||
Relevance: Ensuring users privacy is protected positively influences user’s experience,
|
||||
acceptance and continuous use of the platform. Besides other factors, the reputation of the
|
||||
platform depends on how well user’s information is secure and preserved.
|
||||
|
||||
PRIVACY MEASURES
|
||||
Capture trust requirements
|
||||
Perform risk analysis so measures can be implemented
|
||||
Actions Explore the vulnerability aspects of city data
|
||||
Check whether extensibility requirements impact on trust
|
||||
Define a trust model
|
||||
Allow users to interact with the platform anonymously in given
|
||||
circumstances
|
||||
Manage users identification securely
|
||||
Strategy Anonymize data whenever necessary to comply with data regulations
|
||||
Cryptograph personal identifiers during data transmission when that is
|
||||
necessary
|
||||
Avoid unauthorized access to implicit information (e.g. location)
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 37
|
||||
|
||||
|
||||
Verify the impact of security, trust and scalability requirements trade-offs
|
||||
on privacy
|
||||
Allow the user to control how personal information is used
|
||||
Empower user to control the data disclosure mechanism
|
||||
|
||||
|
||||
|
||||
3.2 Non Run-time Quality Requirements
|
||||
3.2.1 Evolvability Requirements
|
||||
|
||||
Rationale: The ability of the platform to withstand and easily adapt when new requirements and
|
||||
changes is introduced.
|
||||
Drivers: Ensure the platform is able to accommodate additional functionality and emerging
|
||||
technologies at later stage at a fair and transparent cost
|
||||
Refines into requirements: NFREQ.23
|
||||
Relevance: Important for longer- lived and more widely used systems. Urban platforms are
|
||||
expected to be highly evolvable in order to accommodate future emerging technologies and avoid
|
||||
interoperability issues.
|
||||
|
||||
EVOLVABILITY MEASURES
|
||||
Actions Characterize and assess the evolution needs
|
||||
Consider the evolution trade-offs
|
||||
Balance and negotiate potential conflicting requirements emerging from
|
||||
the evolution capability.
|
||||
Strategy Adopt standards and open interfaces
|
||||
Design loose-coupled components
|
||||
Preserve the platform resilience
|
||||
|
||||
|
||||
|
||||
3.2.2 Extensibility Requirements
|
||||
|
||||
Rationale: The flexibility of the system to allow services and functionality to be extended and
|
||||
augmented by service providers in order to increase value of services to both platform providers
|
||||
and end-users. The extension of the platform services is determined by the Platform Openness
|
||||
strategy.
|
||||
Drivers: Stakeholders can extend the services provided by the urban platform, so that
|
||||
partnerships can be built to deliver holistic and interoperable solutions. Identify integrated
|
||||
approaches to design and service delivery which ensures that services fit together and that
|
||||
synergies can be exploited.
|
||||
Refines into requirements: NFREQ.24, NFREQ.25
|
||||
Relevance: Important for longer- lived and more widely used systems. Urban platforms are
|
||||
expected to be highly evolvable in order to accommodate future emerging technologies and avoid
|
||||
interoperability issues.
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 38
|
||||
|
||||
|
||||
EXTENSIBILITY
|
||||
Actions Characterize and assess the extensibility needs
|
||||
Trace the impacts and cost of extensions
|
||||
Negotiate potential conflicting requirements emerging from adding extensions
|
||||
in the platform
|
||||
Strategy Develop a modular architecture with standard interfaces that reduces entry
|
||||
barriers due to increased transparency and integration
|
||||
Share technical information about interfaces will help service providers in
|
||||
targeting opportunities around the platform
|
||||
|
||||
|
||||
|
||||
3.3 List of Non-Functional Requirements
|
||||
|
||||
ID # Description Concern Priority Domain
|
||||
|
||||
Platform,
|
||||
NFREQ.1 Support different service level agreements (SLA) Scalability Should
|
||||
Infrastructure
|
||||
Platform,
|
||||
NFREQ.2 Process services and events on a set of distributed nodes Scalability Should
|
||||
Infrastructure
|
||||
Platform,
|
||||
NFREQ.3 Continuously monitor Quality of Service at runtime Scalability Should
|
||||
Infrastructure
|
||||
Platform,
|
||||
NFREQ.4 Balance its load at runtime Scalability Should
|
||||
Infrastructure
|
||||
Platform,
|
||||
NFREQ.5 Provide high availability Availability Should
|
||||
Infrastructure
|
||||
Platform,
|
||||
NFREQ.6 Guarantee infrastructure availability Availability Should
|
||||
Infrastructure
|
||||
Platform,
|
||||
NFREQ.7 Ensure network availability Availability Should
|
||||
Infrastructure
|
||||
Platform,
|
||||
NFREQ.8 Be able to perform self-healing Availability Should
|
||||
Infrastructure
|
||||
City Data,
|
||||
NFREQ.9 Expose data and services to authorized users Security Must
|
||||
Platform
|
||||
City Data,
|
||||
NFREQ.10 Ensure services are always accessible to entitled users Security Must
|
||||
Platform
|
||||
|
||||
NFREQ.11 Ensure Data Freshness Security Must Platform
|
||||
|
||||
NFREQ.12 Support access control mechanisms Security Must Platform
|
||||
|
||||
Platform,
|
||||
NFREQ.13 Have security mechanisms to protect data transmission Security Should
|
||||
Infrastructure
|
||||
Platform,
|
||||
NFREQ.14 Make it difficult to spy on communicated messages Security Should
|
||||
Infrastructure
|
||||
Platform,
|
||||
NFREQ.15 Be able to perform to detect threats at runtime Security Should
|
||||
Infrastructure
|
||||
Provide trusted and secure communication and information Platform,
|
||||
NFREQ.16 Trust Should
|
||||
management Infrastructure
|
||||
|
||||
NFREQ.17 The platform infrastructure and services shall be trustable Trust Should Infrastructure
|
||||
|
||||
Societal Needs,
|
||||
NFREQ.18 Allow users to use free services anonymously Privacy Should
|
||||
Platform
|
||||
Societal Needs,
|
||||
NFREQ.19 Allow people to use free services anonymously Privacy Should
|
||||
Platform
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 39
|
||||
|
||||
|
||||
Allow users to control which data they are willing to provide Societal Needs,
|
||||
NFREQ.20 Privacy Should
|
||||
and how their data should be used Platform
|
||||
|
||||
NFREQ.21 Keep users access-control rights/ policies secured. Privacy Should Platform
|
||||
|
||||
Provide privacy protection for users interacting with the Societal Needs,
|
||||
NFREQ.22 Privacy Should
|
||||
platform Platform
|
||||
Platform,
|
||||
NFREQ.23 Provide communication confidentiality Privacy Should
|
||||
Infrastructure
|
||||
Societal Needs,
|
||||
NFREQ.24 Be extensible for future technologies. Evolvability Must City Data, Platform,
|
||||
Infrastructure
|
||||
Platform,
|
||||
NFREQ.26 Provide standard interfaces for service providers Extensibility Should
|
||||
Business Needs
|
||||
|
||||
NFREQ.26 Be able to provide services in an interoperable manner Extensibility Should Platform
|
||||
|
||||
|
||||
|
||||
|
||||
4. Other Requirements
|
||||
4.1 Minimal Descriptive Metadata Required from Data Provider
|
||||
|
||||
Minimal Descriptive Metadata Required from Data Provider
|
||||
|
||||
Attribute Required Definition Notes Examples
|
||||
|
||||
Database Name Y A name given to the resource
|
||||
|
||||
Author Name of a person or body associated with
|
||||
Y
|
||||
the creation of the resource
|
||||
|
||||
Maintainer Name of the entity responsible for making
|
||||
Y
|
||||
the resource available
|
||||
|
||||
Date Created Y Date the dataset was created
|
||||
|
||||
|
||||
Date Modified Y Date the dataset was modified
|
||||
|
||||
|
||||
Last Revision Y Date the dataset was last revised
|
||||
|
||||
|
||||
Update Frequency Y Frequency of data maintenance
|
||||
|
||||
|
||||
Mode of Release Y Open/Proprietary
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 40
|
||||
|
||||
|
||||
4.2 Urban Platform Supported Data Formats
|
||||
|
||||
|
||||
MIME type Description Extensions Level
|
||||
|
||||
application/json JavaScript Object Notation json supported
|
||||
|
||||
application/xml eXtensible markup language file xml supported
|
||||
|
||||
text/csv Comma separated values file csv supported
|
||||
|
||||
|
||||
|
||||
|
||||
5. Conclusion & Forward Plans
|
||||
This document represents the first set of requirements specification for urban platform. Future
|
||||
activities will collaboratively assess, resolve requirements conflicts, prioritize, and validate the
|
||||
requirements of the urban platform. Ultimately, this document will become a complete final
|
||||
requirements specification document to guide and speed up the development open platform for
|
||||
cities. Table below shows the milestones, deliverables and engagement activities completed and
|
||||
yet to be completed by the Demand Side Engagement Stream.
|
||||
|
||||
|
||||
|
||||
Milestones, deliverables and engagement activity Forecast date Status
|
||||
Stakeholders invitation Late Completed
|
||||
September/2015
|
||||
Online Workshop 1: Project Scope and Outcomes 20/11/2015 Completed
|
||||
Online meeting held with city members to communicate goals,
|
||||
expected outcomes, time commitment and required actions.
|
||||
Start of Collaborative Requirements Engineering Process 20/11/2015 Completed
|
||||
Participants reviewed the first draft of urban platform requirements,
|
||||
provided comments, and suggested new requirements and changes.
|
||||
Online Workshop 2: Review and Refine Requirements 08/12/2015 Completed
|
||||
Participants had a conference call to review the requirements
|
||||
specification document and provide their comments.
|
||||
Deliverable: Requirements Specification Document v2.1 shared 05/01/2016 Completed
|
||||
across the Working Streams for consultation
|
||||
Online Workshop 3: Review Requirements Specification 12/01/2016 Completed
|
||||
Document and discussion of Letter of Intent
|
||||
Participants had a conference call to review the requirements
|
||||
specification document and provide their comments, and discussed
|
||||
the Letter of Intent to be signed by Mayor/Equiv of EU cities to
|
||||
commit to application of common U.P. approach.
|
||||
Urban Platforms Workshop in Brussels 19/01/2016 Completed
|
||||
Participants provided feedback on the requirements specification
|
||||
document and recommended changes in the document
|
||||
Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 41
|
||||
|
||||
|
||||
Deliverable: Updated Requirements Specification Document v2.2 29/01/2016 Completed
|
||||
shared with Industry and Standardisation Working Streams
|
||||
Deliverable: Online release of the Requirements Specification Early
|
||||
Document and open calls for wider consultation on the Requirements February/2016
|
||||
|
||||
Online Workshop 4: Engagement with Industry Side Early
|
||||
Capture industry feedback on the Requirements Specification February/2016
|
||||
Document to assess its suitability to drive the development of the first
|
||||
draft reference architecture.
|
||||
Online Workshop 5: Review Requirements Specification Mid
|
||||
Document v2.2 and discuss requirement prioritization and February/2016
|
||||
balancing
|
||||
Collectively determine which candidate requirements of urban
|
||||
platforms should be prioritized as high importance to all the cities
|
||||
(use of Value Oriented Prioritization Method). Requirements are also
|
||||
prioritized to minimize risk during development of urban platforms so
|
||||
that the most important requirements are made explicit and
|
||||
considered in the reference architecture.
|
||||
Capture of case studies from cities intended to support requirements Mid March/2016
|
||||
validation (City of Porto), and learning and confidence building
|
||||
amongst cities.
|
||||
2455
files/extracted_requirements.txt
Normal file
2455
files/extracted_requirements.txt
Normal file
File diff suppressed because it is too large
Load Diff
@@ -3,6 +3,7 @@ package agent
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -32,6 +33,29 @@ type Orchestrator struct {
|
||||
enableCapabilityGap bool
|
||||
log *logger.Logger
|
||||
skillsMu sync.RWMutex
|
||||
pendingFilesMu sync.Mutex
|
||||
pendingFiles map[string][]pendingFileRef
|
||||
}
|
||||
|
||||
type pendingFileRef struct {
|
||||
ID string
|
||||
Name string
|
||||
MimeType string
|
||||
}
|
||||
|
||||
type capabilityRoutingResult struct {
|
||||
NeedSkills bool
|
||||
SelectedToolNames []string
|
||||
SelectedSkills []knowledge.Skill
|
||||
Reason string
|
||||
UsedFallback bool
|
||||
}
|
||||
|
||||
type filePromptContext struct {
|
||||
Summary string
|
||||
FatalReason string
|
||||
FileIDs []string
|
||||
Uploaded []pendingFileRef
|
||||
}
|
||||
|
||||
// NewOrchestrator 创建一个新的编排器对象，初始化关键路径和超时控制等。
|
||||
@@ -76,6 +100,7 @@ func NewOrchestrator(
|
||||
reactMaxStep: reactMaxStep,
|
||||
enableCapabilityGap: enableCapabilityGap,
|
||||
log: log,
|
||||
pendingFiles: make(map[string][]pendingFileRef),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,12 +110,20 @@ func NewOrchestrator(
|
||||
// - 是否需要调用工具（action + action_input）
|
||||
// 循环持续进行，直到 LLM 返回 is_final_answer=true。
|
||||
func (o *Orchestrator) HandleMessage(ctx context.Context, chatID, userID, text string) (string, error) {
|
||||
return o.handleMessageInternal(ctx, chatID, userID, text, nil)
|
||||
}
|
||||
|
||||
func (o *Orchestrator) HandleMessageWithFiles(ctx context.Context, chatID, userID, text string, files []llm.InputFile) (string, error) {
|
||||
return o.handleMessageInternal(ctx, chatID, userID, text, files)
|
||||
}
|
||||
|
||||
func (o *Orchestrator) handleMessageInternal(ctx context.Context, chatID, userID, text string, files []llm.InputFile) (string, error) {
|
||||
// 为链路追踪设置唯一的 TraceID
|
||||
traceID := logger.NewTraceID()
|
||||
ctx = logger.WithTraceID(ctx, traceID)
|
||||
traceLogPrefix := "trace_id=" + traceID
|
||||
if o.log != nil {
|
||||
o.log.Infof("%s handle message chat_id=%s user_id=%s text_len=%d", traceLogPrefix, chatID, userID, len(text))
|
||||
o.log.Infof("%s handle message chat_id=%s user_id=%s text_len=%d files=%d", traceLogPrefix, chatID, userID, len(text), len(files))
|
||||
o.log.Debugf("%s handle message text=%q", traceLogPrefix, text)
|
||||
}
|
||||
|
||||
@@ -111,6 +144,38 @@ func (o *Orchestrator) HandleMessage(ctx context.Context, chatID, userID, text s
|
||||
return report, nil
|
||||
}
|
||||
|
||||
trimmedText := strings.TrimSpace(text)
|
||||
isFileOnly := len(files) > 0 && trimmedText == ""
|
||||
|
||||
if isFileOnly {
|
||||
if err := o.store.SaveMessage(chatID, userID, "user", "[FILE_UPLOAD]"); err != nil {
|
||||
if o.log != nil {
|
||||
o.log.Errorf("%s save file-only user marker failed chat_id=%s err=%v", traceLogPrefix, chatID, err)
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
uploadCtx := o.prepareFilePromptContext(ctx, files, nil)
|
||||
if strings.TrimSpace(uploadCtx.FatalReason) != "" {
|
||||
finalText := "文件上传失败，无法建立文档上下文。" + "\n" + uploadCtx.FatalReason
|
||||
if err := o.store.SaveMessage(chatID, userID, "assistant", finalText); err != nil && o.log != nil {
|
||||
o.log.Warnf("%s save upload failure message failed chat_id=%s err=%v", traceLogPrefix, chatID, err)
|
||||
}
|
||||
return finalText, nil
|
||||
}
|
||||
o.appendPendingFiles(chatID, userID, uploadCtx.toPendingRefs())
|
||||
finalText := o.buildFileUploadAck(uploadCtx)
|
||||
if err := o.store.SaveMessage(chatID, userID, "assistant", finalText); err != nil {
|
||||
if o.log != nil {
|
||||
o.log.Errorf("%s save file upload ack failed chat_id=%s err=%v", traceLogPrefix, chatID, err)
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
if o.log != nil {
|
||||
o.log.Infof("%s file-only message handled chat_id=%s cached_files=%d", traceLogPrefix, chatID, len(uploadCtx.FileIDs))
|
||||
}
|
||||
return finalText, nil
|
||||
}
|
||||
|
||||
// 保存用户消息到 SQLite 中
|
||||
if err := o.store.SaveMessage(chatID, userID, "user", text); err != nil {
|
||||
if o.log != nil {
|
||||
@@ -133,13 +198,30 @@ func (o *Orchestrator) HandleMessage(ctx context.Context, chatID, userID, text s
|
||||
}
|
||||
|
||||
// 进入统一 ReAct 循环
|
||||
response, err := o.runUnifiedReAct(ctx, chatID, userID, compressed, text)
|
||||
pendingRefs := o.getPendingFiles(chatID, userID)
|
||||
fileCtx := o.prepareFilePromptContext(ctx, files, pendingRefs)
|
||||
if strings.TrimSpace(fileCtx.FatalReason) != "" {
|
||||
finalText := "文件上传失败，无法继续进行文档解析。" + "\n" + fileCtx.FatalReason
|
||||
if err := o.store.SaveMessage(chatID, userID, "assistant", finalText); err != nil && o.log != nil {
|
||||
o.log.Warnf("%s save assistant failure message failed chat_id=%s err=%v", traceLogPrefix, chatID, err)
|
||||
}
|
||||
if o.log != nil {
|
||||
o.log.Warnf("%s stop before react due to file upload failure reason=%s", traceLogPrefix, fileCtx.FatalReason)
|
||||
}
|
||||
return finalText, nil
|
||||
}
|
||||
routeInput := composeRouteInput(text, fileCtx.Summary)
|
||||
route := o.routeCapabilities(ctx, routeInput)
|
||||
response, err := o.runUnifiedReAct(ctx, chatID, userID, compressed, text, fileCtx, routeInput, route)
|
||||
if err != nil {
|
||||
if o.log != nil {
|
||||
o.log.Errorf("%s message generation failed chat_id=%s err=%v", traceLogPrefix, chatID, err)
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
if len(pendingRefs) > 0 {
|
||||
o.clearPendingFiles(chatID, userID)
|
||||
}
|
||||
|
||||
// 最终将机器人的回复也加入记忆缓存
|
||||
if err := o.store.SaveMessage(chatID, userID, "assistant", response); err != nil {
|
||||
@@ -156,21 +238,29 @@ func (o *Orchestrator) HandleMessage(ctx context.Context, chatID, userID, text s
|
||||
}
|
||||
|
||||
// buildUnifiedSystemPrompt 构建统一 ReAct 循环的 system prompt。
|
||||
// 包含人格设定、所有可用技能（含完整内容）、所有可用工具、以及 JSON 输出格式约束。
|
||||
func (o *Orchestrator) buildUnifiedSystemPrompt() string {
|
||||
// 工具始终可用；技能仅按当前问题挑选相关项作为增强上下文。
|
||||
func (o *Orchestrator) buildUnifiedSystemPrompt(userInput string, route capabilityRoutingResult) string {
|
||||
skillMetaDoc := o.formatSkillSummariesForPrompt()
|
||||
allSkillsDoc := o.formatAllSkillsContent()
|
||||
relevantSkillsDoc := o.formatSelectedSkillsForPrompt(userInput, route.SelectedSkills)
|
||||
toolDoc := o.formatToolDoc()
|
||||
runtimeDoc := formatRuntimeContextForPrompt()
|
||||
routeDoc := formatRouteForPrompt(route)
|
||||
|
||||
return strings.Join([]string{
|
||||
"你是一个个人自动化助手，必须遵循如下人格设定并保持一致：",
|
||||
o.soul,
|
||||
"",
|
||||
"===== 运行环境 =====",
|
||||
runtimeDoc,
|
||||
"",
|
||||
"===== 可用技能概览 =====",
|
||||
skillMetaDoc,
|
||||
"",
|
||||
"===== 技能详细说明 =====",
|
||||
allSkillsDoc,
|
||||
"===== 能力路由结果 =====",
|
||||
routeDoc,
|
||||
"",
|
||||
"===== 本轮相关技能（按用户问题筛选） =====",
|
||||
relevantSkillsDoc,
|
||||
"",
|
||||
"===== 可用工具 =====",
|
||||
toolDoc,
|
||||
@@ -190,25 +280,32 @@ func (o *Orchestrator) buildUnifiedSystemPrompt() string {
|
||||
"决策规则：",
|
||||
"1) 如果你可以直接回答用户问题（不需要任何工具）：",
|
||||
" 设 is_final_answer=true，action=\"none\"，final_answer 填写完整回复。",
|
||||
"2) 如果你需要调用工具获取信息后才能回答：",
|
||||
"2) 优先判断是否可通过原子工具能力完成任务；若可完成，直接进行工具调用链路。",
|
||||
"3) 当纯工具调用无法满足时，再结合已加载的技能详细说明进行决策。",
|
||||
"4) 如果你需要调用工具获取信息后才能回答：",
|
||||
" 设 is_final_answer=false，action 填工具名，action_input 填工具所需输入，final_answer=null。",
|
||||
"3) 不要在 JSON 之外输出任何内容。",
|
||||
"4) 根据技能说明中的指引决定何时以及如何使用工具。",
|
||||
"5) 每轮工具调用结果会以 Observation 的形式追加到推理记录中，供你下一轮决策参考。",
|
||||
"5) 不要在 JSON 之外输出任何内容。",
|
||||
"6) 根据技能说明中的指引决定何时以及如何使用工具。",
|
||||
"7) 工具能力是全局可用的，不依赖技能命中；当技能不匹配时，仍可直接选择合适工具。",
|
||||
"8) 若技能中存在与当前运行环境不匹配的章节（如 Windows 专章），应降低优先级，除非用户明确要求该环境。",
|
||||
"9) 每轮工具调用结果会以 Observation 的形式追加到推理记录中，供你下一轮决策参考。",
|
||||
}, "\n")
|
||||
}
|
||||
|
||||
// runUnifiedReAct 执行统一的 ReAct 循环。
|
||||
// LLM 每次都看到完整的技能集+工具集，自行决定是否调用工具或直接回答。
|
||||
// 循环持续到 is_final_answer=true 或达到安全上限。
|
||||
func (o *Orchestrator) runUnifiedReAct(ctx context.Context, chatID, userID, compressedContext, userInput string) (string, error) {
|
||||
func (o *Orchestrator) runUnifiedReAct(ctx context.Context, chatID, userID, compressedContext, userInput string, fileCtx filePromptContext, routeInput string, route capabilityRoutingResult) (string, error) {
|
||||
traceID := logger.TraceIDFromContext(ctx)
|
||||
traceLogPrefix := "trace_id=" + traceID
|
||||
|
||||
systemPrompt := o.buildUnifiedSystemPrompt()
|
||||
if strings.TrimSpace(routeInput) == "" {
|
||||
routeInput = composeRouteInput(userInput, fileCtx.Summary)
|
||||
}
|
||||
systemPrompt := o.buildUnifiedSystemPrompt(routeInput, route)
|
||||
|
||||
if o.log != nil {
|
||||
o.log.Infof("%s unified react start", traceLogPrefix)
|
||||
o.log.Infof("%s unified react start route_need_skills=%v route_tools=%v route_skills=%d fallback=%v", traceLogPrefix, route.NeedSkills, route.SelectedToolNames, len(route.SelectedSkills), route.UsedFallback)
|
||||
}
|
||||
|
||||
// 安全上限：防止无限循环（当前暂不使用 reactMaxStep 配置约束，使用固定硬上限）
|
||||
@@ -229,13 +326,16 @@ func (o *Orchestrator) runUnifiedReAct(ctx context.Context, chatID, userID, comp
|
||||
"用户问题：",
|
||||
userInput,
|
||||
"",
|
||||
"文件上下文：",
|
||||
defaultIfEmpty(fileCtx.Summary, "(none)"),
|
||||
"",
|
||||
"当前推理记录（按时间顺序）：",
|
||||
scratchpad,
|
||||
"",
|
||||
"请输出你的 JSON 决策。",
|
||||
}, "\n")
|
||||
|
||||
raw, err := o.llm.Generate(ctx, systemPrompt, prompt)
|
||||
raw, err := o.generateWithOptionalFiles(ctx, systemPrompt, prompt, fileCtx.FileIDs)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -335,15 +435,514 @@ func (o *Orchestrator) runUnifiedReAct(ctx context.Context, chatID, userID, comp
|
||||
return "我尝试了多轮推理与工具调用，但仍未得到稳定结论。请给我更具体的约束或允许我继续尝试。", nil
|
||||
}
|
||||
|
||||
// formatAllSkillsContent 返回所有技能的完整内容，用于注入到 system prompt 中。
|
||||
func (o *Orchestrator) formatAllSkillsContent() string {
|
||||
skills := o.getSkillsSnapshot()
|
||||
func composeRouteInput(userInput, fileSummary string) string {
|
||||
userInput = strings.TrimSpace(userInput)
|
||||
fileSummary = strings.TrimSpace(fileSummary)
|
||||
if userInput == "" {
|
||||
return fileSummary
|
||||
}
|
||||
if fileSummary == "" {
|
||||
return userInput
|
||||
}
|
||||
return userInput + "\n\n" + fileSummary
|
||||
}
|
||||
|
||||
func (o *Orchestrator) prepareFilePromptContext(ctx context.Context, files []llm.InputFile, pending []pendingFileRef) filePromptContext {
|
||||
ctxOut := filePromptContext{}
|
||||
if len(pending) > 0 {
|
||||
for _, p := range pending {
|
||||
id := strings.TrimSpace(p.ID)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
ctxOut.FileIDs = append(ctxOut.FileIDs, id)
|
||||
}
|
||||
}
|
||||
if len(files) == 0 {
|
||||
ctxOut.Summary = buildFileSummary(pending, nil)
|
||||
return ctxOut
|
||||
}
|
||||
uploader, ok := o.llm.(llm.FileUploader)
|
||||
if !ok {
|
||||
return filePromptContext{FatalReason: "检测到文件输入，但当前 LLM 客户端不支持文件上传接口。"}
|
||||
}
|
||||
|
||||
uploaded := make([]pendingFileRef, 0, len(files))
|
||||
for i, f := range files {
|
||||
if strings.TrimSpace(f.FileName) == "" || len(f.Content) == 0 {
|
||||
return filePromptContext{FatalReason: fmt.Sprintf("file[%d] 缺少文件名或内容，无法上传。", i+1)}
|
||||
}
|
||||
fileID, err := uploader.UploadFile(ctx, f, "file-extract")
|
||||
if err != nil {
|
||||
return filePromptContext{FatalReason: fmt.Sprintf("file[%d] name=%s 上传失败: %v", i+1, f.FileName, err)}
|
||||
}
|
||||
ctxOut.FileIDs = append(ctxOut.FileIDs, fileID)
|
||||
uploaded = append(uploaded, pendingFileRef{
|
||||
ID: fileID,
|
||||
Name: strings.TrimSpace(f.FileName),
|
||||
MimeType: defaultIfEmpty(strings.TrimSpace(f.MimeType), "application/octet-stream"),
|
||||
})
|
||||
}
|
||||
ctxOut.Uploaded = uploaded
|
||||
ctxOut.Summary = buildFileSummary(pending, uploaded)
|
||||
return ctxOut
|
||||
}
|
||||
|
||||
func buildFileSummary(pending, uploaded []pendingFileRef) string {
|
||||
if len(pending) == 0 && len(uploaded) == 0 {
|
||||
return ""
|
||||
}
|
||||
lines := make([]string, 0, len(pending)+len(uploaded)+2)
|
||||
lines = append(lines, "以下文件 file_id 可用于本轮问答：")
|
||||
idx := 1
|
||||
for _, p := range pending {
|
||||
id := strings.TrimSpace(p.ID)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("- cached_file[%d] name=%s mime=%s file_id=%s", idx, defaultIfEmpty(strings.TrimSpace(p.Name), "(unknown)"), defaultIfEmpty(strings.TrimSpace(p.MimeType), "application/octet-stream"), id))
|
||||
idx++
|
||||
}
|
||||
for _, p := range uploaded {
|
||||
id := strings.TrimSpace(p.ID)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("- uploaded_file[%d] name=%s mime=%s file_id=%s", idx, defaultIfEmpty(strings.TrimSpace(p.Name), "(unknown)"), defaultIfEmpty(strings.TrimSpace(p.MimeType), "application/octet-stream"), id))
|
||||
idx++
|
||||
}
|
||||
if len(lines) == 1 {
|
||||
return ""
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
func (o *Orchestrator) generateWithOptionalFiles(ctx context.Context, systemPrompt, userPrompt string, fileIDs []string) (string, error) {
|
||||
ids := nonEmptyIDs(fileIDs)
|
||||
if len(ids) == 0 {
|
||||
return o.llm.Generate(ctx, systemPrompt, userPrompt)
|
||||
}
|
||||
client, ok := o.llm.(llm.FileChatClient)
|
||||
if !ok {
|
||||
return o.llm.Generate(ctx, systemPrompt, userPrompt)
|
||||
}
|
||||
return client.GenerateWithFiles(ctx, systemPrompt, userPrompt, ids)
|
||||
}
|
||||
|
||||
func (o *Orchestrator) buildFileUploadAck(ctx filePromptContext) string {
|
||||
if len(ctx.FileIDs) == 0 {
|
||||
return "文件已接收，但未拿到有效 file_id。请重新上传一次。"
|
||||
}
|
||||
lines := []string{
|
||||
fmt.Sprintf("文件上传完成，已缓存 %d 个 file_id。", len(ctx.FileIDs)),
|
||||
"请继续发送你的问题，我会结合这些文件内容和历史对话一起回答。",
|
||||
}
|
||||
if strings.TrimSpace(ctx.Summary) != "" {
|
||||
lines = append(lines, "", ctx.Summary)
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
func nonEmptyIDs(ids []string) []string {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(ids))
|
||||
seen := map[string]struct{}{}
|
||||
for _, id := range ids {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
out = append(out, id)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (c filePromptContext) toPendingRefs() []pendingFileRef {
|
||||
if len(c.Uploaded) > 0 {
|
||||
copied := make([]pendingFileRef, len(c.Uploaded))
|
||||
copy(copied, c.Uploaded)
|
||||
return sanitizePendingRefs(copied)
|
||||
}
|
||||
ids := nonEmptyIDs(c.FileIDs)
|
||||
out := make([]pendingFileRef, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
out = append(out, pendingFileRef{ID: id})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (o *Orchestrator) appendPendingFiles(chatID, userID string, refs []pendingFileRef) {
|
||||
refs = sanitizePendingRefs(refs)
|
||||
if len(refs) == 0 {
|
||||
return
|
||||
}
|
||||
key := pendingFileKey(chatID, userID)
|
||||
o.pendingFilesMu.Lock()
|
||||
defer o.pendingFilesMu.Unlock()
|
||||
merged := append(o.pendingFiles[key], refs...)
|
||||
o.pendingFiles[key] = sanitizePendingRefs(merged)
|
||||
}
|
||||
|
||||
func (o *Orchestrator) getPendingFiles(chatID, userID string) []pendingFileRef {
|
||||
key := pendingFileKey(chatID, userID)
|
||||
o.pendingFilesMu.Lock()
|
||||
defer o.pendingFilesMu.Unlock()
|
||||
snapshot := o.pendingFiles[key]
|
||||
out := make([]pendingFileRef, len(snapshot))
|
||||
copy(out, snapshot)
|
||||
return out
|
||||
}
|
||||
|
||||
func (o *Orchestrator) clearPendingFiles(chatID, userID string) {
|
||||
key := pendingFileKey(chatID, userID)
|
||||
o.pendingFilesMu.Lock()
|
||||
defer o.pendingFilesMu.Unlock()
|
||||
delete(o.pendingFiles, key)
|
||||
}
|
||||
|
||||
func pendingFileKey(chatID, userID string) string {
|
||||
return strings.TrimSpace(chatID) + "::" + strings.TrimSpace(userID)
|
||||
}
|
||||
|
||||
func sanitizePendingRefs(refs []pendingFileRef) []pendingFileRef {
|
||||
if len(refs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]pendingFileRef, 0, len(refs))
|
||||
seen := map[string]struct{}{}
|
||||
for _, r := range refs {
|
||||
id := strings.TrimSpace(r.ID)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
r.ID = id
|
||||
r.Name = strings.TrimSpace(r.Name)
|
||||
r.MimeType = strings.TrimSpace(r.MimeType)
|
||||
out = append(out, r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func defaultIfEmpty(v, fallback string) string {
|
||||
if strings.TrimSpace(v) == "" {
|
||||
return fallback
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// formatRelevantSkillsForPrompt 返回与当前用户问题最相关的技能内容。
|
||||
func (o *Orchestrator) formatSelectedSkillsForPrompt(userInput string, selected []knowledge.Skill) string {
|
||||
skills := selected
|
||||
if len(skills) == 0 {
|
||||
return "(none)"
|
||||
skills = o.selectRelevantSkills(userInput, 4)
|
||||
}
|
||||
if len(skills) == 0 {
|
||||
return "(none matched, tools are still globally available)"
|
||||
}
|
||||
return formatSkills(skills)
|
||||
}
|
||||
|
||||
func (o *Orchestrator) routeCapabilities(ctx context.Context, userInput string) capabilityRoutingResult {
|
||||
fallback := capabilityRoutingResult{
|
||||
NeedSkills: true,
|
||||
SelectedSkills: o.selectRelevantSkills(userInput, 4),
|
||||
Reason: "router fallback: keyword matching",
|
||||
UsedFallback: true,
|
||||
}
|
||||
|
||||
raw, err := o.llm.Generate(ctx, o.buildRouteSystemPrompt(), o.buildRouteUserPrompt(userInput))
|
||||
if err != nil {
|
||||
if o.log != nil {
|
||||
o.log.Warnf("capability router llm call failed err=%v", err)
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
decision, err := parseCapabilityRoute(raw)
|
||||
if err != nil {
|
||||
if o.log != nil {
|
||||
o.log.Warnf("capability router parse failed err=%v raw=%q", err, raw)
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
resolvedTools := o.normalizeToolSelection(decision.SelectedTools)
|
||||
resolved := capabilityRoutingResult{
|
||||
NeedSkills: decision.NeedSkills,
|
||||
SelectedToolNames: resolvedTools,
|
||||
Reason: strings.TrimSpace(decision.Reason),
|
||||
}
|
||||
|
||||
if resolved.NeedSkills {
|
||||
skills := o.resolveSkillsByNames(decision.SelectedSkills, 4)
|
||||
if len(skills) == 0 {
|
||||
skills = o.selectRelevantSkills(userInput, 4)
|
||||
resolved.UsedFallback = true
|
||||
}
|
||||
resolved.SelectedSkills = skills
|
||||
}
|
||||
|
||||
return resolved
|
||||
}
|
||||
|
||||
func (o *Orchestrator) buildRouteSystemPrompt() string {
|
||||
return strings.Join([]string{
|
||||
"你是能力路由器（Router Agent）。",
|
||||
"你的任务是：在不加载技能全文的前提下，仅根据工具摘要和技能摘要，判断本请求是否可以仅靠原子工具能力完成，还是需要加载技能详细说明。",
|
||||
"输出必须且仅能是 JSON：",
|
||||
"{",
|
||||
" \"need_skills\": true 或 false,",
|
||||
" \"selected_tools\": [\"tool_name\", ...],",
|
||||
" \"selected_skills\": [\"skill_name\", ...],",
|
||||
" \"reason\": \"简短路由理由\"",
|
||||
"}",
|
||||
"规则：",
|
||||
"1) 优先原子工具能力。若可通过工具链路完成，need_skills=false。",
|
||||
"2) 只有当工具能力不足以覆盖业务约束时，need_skills=true 并选择少量最相关技能。",
|
||||
"3) selected_skills 仅填写技能名称（来自技能摘要）。",
|
||||
"4) selected_tools 仅填写可用工具名。",
|
||||
"5) 不要输出 JSON 之外内容。",
|
||||
}, "\n")
|
||||
}
|
||||
|
||||
func (o *Orchestrator) buildRouteUserPrompt(userInput string) string {
|
||||
return strings.Join([]string{
|
||||
"当前运行环境：",
|
||||
formatRuntimeContextForPrompt(),
|
||||
"",
|
||||
"用户问题：",
|
||||
userInput,
|
||||
"",
|
||||
"可用工具摘要：",
|
||||
o.formatToolDoc(),
|
||||
"",
|
||||
"可用技能摘要：",
|
||||
o.formatSkillSummariesForPrompt(),
|
||||
"",
|
||||
"请给出路由 JSON。",
|
||||
}, "\n")
|
||||
}
|
||||
|
||||
func (o *Orchestrator) normalizeToolSelection(in []string) []string {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
allowed := map[string]struct{}{}
|
||||
for _, t := range o.tools.List() {
|
||||
allowed[strings.ToLower(strings.TrimSpace(t.Name()))] = struct{}{}
|
||||
}
|
||||
out := make([]string, 0, len(in))
|
||||
set := map[string]struct{}{}
|
||||
for _, name := range in {
|
||||
n := strings.ToLower(strings.TrimSpace(name))
|
||||
if n == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := allowed[n]; !ok {
|
||||
continue
|
||||
}
|
||||
if _, exists := set[n]; exists {
|
||||
continue
|
||||
}
|
||||
set[n] = struct{}{}
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (o *Orchestrator) resolveSkillsByNames(names []string, maxCount int) []knowledge.Skill {
|
||||
if len(names) == 0 {
|
||||
return nil
|
||||
}
|
||||
if maxCount <= 0 {
|
||||
maxCount = 4
|
||||
}
|
||||
all := o.getSkillsSnapshot()
|
||||
idx := make(map[string]knowledge.Skill, len(all))
|
||||
for _, sk := range all {
|
||||
key := strings.ToLower(strings.TrimSpace(sk.Name))
|
||||
if key != "" {
|
||||
idx[key] = sk
|
||||
}
|
||||
}
|
||||
out := make([]knowledge.Skill, 0, maxCount)
|
||||
used := map[string]struct{}{}
|
||||
for _, name := range names {
|
||||
key := strings.ToLower(strings.TrimSpace(name))
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
sk, ok := idx[key]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, exists := used[key]; exists {
|
||||
continue
|
||||
}
|
||||
used[key] = struct{}{}
|
||||
out = append(out, sk)
|
||||
if len(out) >= maxCount {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func formatRouteForPrompt(route capabilityRoutingResult) string {
|
||||
b := strings.Builder{}
|
||||
if route.UsedFallback {
|
||||
b.WriteString("router_status: fallback\n")
|
||||
} else {
|
||||
b.WriteString("router_status: ok\n")
|
||||
}
|
||||
b.WriteString("need_skills: ")
|
||||
b.WriteString(strconv.FormatBool(route.NeedSkills))
|
||||
b.WriteString("\n")
|
||||
b.WriteString("selected_tools: ")
|
||||
if len(route.SelectedToolNames) == 0 {
|
||||
b.WriteString("(none)")
|
||||
} else {
|
||||
b.WriteString(strings.Join(route.SelectedToolNames, ", "))
|
||||
}
|
||||
b.WriteString("\n")
|
||||
b.WriteString("selected_skill_count: ")
|
||||
b.WriteString(strconv.Itoa(len(route.SelectedSkills)))
|
||||
b.WriteString("\n")
|
||||
if strings.TrimSpace(route.Reason) != "" {
|
||||
b.WriteString("reason: ")
|
||||
b.WriteString(strings.TrimSpace(route.Reason))
|
||||
}
|
||||
return strings.TrimSpace(b.String())
|
||||
}
|
||||
|
||||
func (o *Orchestrator) selectRelevantSkills(userInput string, maxCount int) []knowledge.Skill {
|
||||
if maxCount <= 0 {
|
||||
maxCount = 4
|
||||
}
|
||||
query := strings.TrimSpace(strings.ToLower(userInput))
|
||||
all := o.getSkillsSnapshot()
|
||||
if query == "" || len(all) <= maxCount {
|
||||
return all
|
||||
}
|
||||
|
||||
queryTokens := buildQueryTokens(query)
|
||||
type item struct {
|
||||
skill knowledge.Skill
|
||||
score int
|
||||
}
|
||||
ranked := make([]item, 0, len(all))
|
||||
|
||||
for _, sk := range all {
|
||||
hay := strings.ToLower(sk.Name + "\n" + clipForScoring(sk.Content, 1800))
|
||||
score := 0
|
||||
if strings.Contains(hay, query) {
|
||||
score += 8
|
||||
}
|
||||
for _, tk := range queryTokens {
|
||||
if strings.Contains(hay, tk) {
|
||||
score++
|
||||
}
|
||||
}
|
||||
if score == 0 {
|
||||
continue
|
||||
}
|
||||
ranked = append(ranked, item{skill: sk, score: score})
|
||||
}
|
||||
|
||||
if len(ranked) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
sort.Slice(ranked, func(i, j int) bool {
|
||||
if ranked[i].score == ranked[j].score {
|
||||
return strings.ToLower(strings.TrimSpace(ranked[i].skill.Name)) < strings.ToLower(strings.TrimSpace(ranked[j].skill.Name))
|
||||
}
|
||||
return ranked[i].score > ranked[j].score
|
||||
})
|
||||
|
||||
if len(ranked) > maxCount {
|
||||
ranked = ranked[:maxCount]
|
||||
}
|
||||
out := make([]knowledge.Skill, 0, len(ranked))
|
||||
for _, r := range ranked {
|
||||
out = append(out, r.skill)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func buildQueryTokens(query string) []string {
|
||||
set := map[string]struct{}{}
|
||||
collectToken := func(t string) {
|
||||
t = strings.TrimSpace(t)
|
||||
if len([]rune(t)) < 2 {
|
||||
return
|
||||
}
|
||||
set[t] = struct{}{}
|
||||
}
|
||||
|
||||
for _, part := range strings.FieldsFunc(query, func(r rune) bool {
|
||||
if r >= 'a' && r <= 'z' {
|
||||
return false
|
||||
}
|
||||
if r >= '0' && r <= '9' {
|
||||
return false
|
||||
}
|
||||
if r >= 0x4e00 && r <= 0x9fff {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}) {
|
||||
collectToken(part)
|
||||
}
|
||||
|
||||
// 针对中文无空格输入，补充 2-gram 提升匹配命中率。
|
||||
runes := []rune(query)
|
||||
for i := 0; i+1 < len(runes); i++ {
|
||||
r1 := runes[i]
|
||||
r2 := runes[i+1]
|
||||
if (r1 >= 0x4e00 && r1 <= 0x9fff) && (r2 >= 0x4e00 && r2 <= 0x9fff) {
|
||||
collectToken(string([]rune{r1, r2}))
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]string, 0, len(set))
|
||||
for tk := range set {
|
||||
out = append(out, tk)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func clipForScoring(s string, maxRunes int) string {
|
||||
if maxRunes <= 0 {
|
||||
maxRunes = 1800
|
||||
}
|
||||
r := []rune(s)
|
||||
if len(r) <= maxRunes {
|
||||
return s
|
||||
}
|
||||
return string(r[:maxRunes])
|
||||
}
|
||||
|
||||
func formatRuntimeContextForPrompt() string {
|
||||
goos := strings.TrimSpace(strings.ToLower(runtime.GOOS))
|
||||
if goos == "" {
|
||||
goos = "unknown"
|
||||
}
|
||||
return "当前运行系统 GOOS=" + goos + "。请优先使用与该系统一致的策略。仅当用户明确要求时，才采用其他系统（如 Windows）的专用流程。"
|
||||
}
|
||||
|
||||
// emitCapabilityGap 处理能力缺口信息埋点或者通过 AI 自动创建生成相应缺失技能的逻辑
|
||||
func (o *Orchestrator) emitCapabilityGap(chatID, userID, intent, reason string) {
|
||||
if !o.enableCapabilityGap {
|
||||
|
||||
48
internal/agent/orchestrator_skill_selection_test.go
Normal file
48
internal/agent/orchestrator_skill_selection_test.go
Normal file
@@ -0,0 +1,48 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"laodingbot/internal/knowledge"
|
||||
)
|
||||
|
||||
func TestBuildQueryTokensIncludesChineseBigrams(t *testing.T) {
|
||||
tokens := buildQueryTokens("请执行命令并查看文件")
|
||||
joined := strings.Join(tokens, ",")
|
||||
if !strings.Contains(joined, "命令") {
|
||||
t.Fatalf("expected token contains 命令, got: %v", tokens)
|
||||
}
|
||||
if !strings.Contains(joined, "文件") {
|
||||
t.Fatalf("expected token contains 文件, got: %v", tokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectRelevantSkillsPrefersMatchingSkill(t *testing.T) {
|
||||
o := &Orchestrator{
|
||||
skills: []knowledge.Skill{
|
||||
{Name: "文件系统查询专家", Content: "适用于目录、文件、路径、命令执行等场景"},
|
||||
{Name: "天气查询", Content: "用于天气和空气质量查询"},
|
||||
{Name: "日程助手", Content: "用于日程管理"},
|
||||
},
|
||||
}
|
||||
|
||||
selected := o.selectRelevantSkills("帮我执行命令查看某个文件", 2)
|
||||
if len(selected) == 0 {
|
||||
t.Fatal("expected non-empty selected skills")
|
||||
}
|
||||
if selected[0].Name != "文件系统查询专家" {
|
||||
t.Fatalf("expected top skill 文件系统查询专家, got: %s", selected[0].Name)
|
||||
}
|
||||
if len(selected) > 2 {
|
||||
t.Fatalf("expected at most 2 skills, got: %d", len(selected))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatRuntimeContextForPromptIncludesGOOS(t *testing.T) {
|
||||
doc := formatRuntimeContextForPrompt()
|
||||
if !strings.Contains(strings.ToLower(doc), strings.ToLower(runtime.GOOS)) {
|
||||
t.Fatalf("expected runtime context contains GOOS=%s, got: %s", runtime.GOOS, doc)
|
||||
}
|
||||
}
|
||||
31
internal/agent/router_parser.go
Normal file
31
internal/agent/router_parser.go
Normal file
@@ -0,0 +1,31 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type capabilityRouteDecision struct {
|
||||
NeedSkills bool `json:"need_skills"`
|
||||
SelectedTools []string `json:"selected_tools"`
|
||||
SelectedSkills []string `json:"selected_skills"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
func parseCapabilityRoute(raw string) (capabilityRouteDecision, error) {
|
||||
raw = normalizeJSON(raw)
|
||||
start := strings.Index(raw, "{")
|
||||
end := strings.LastIndex(raw, "}")
|
||||
if start < 0 || end < start {
|
||||
return capabilityRouteDecision{}, fmt.Errorf("no json object found")
|
||||
}
|
||||
raw = raw[start : end+1]
|
||||
|
||||
var out capabilityRouteDecision
|
||||
if err := json.Unmarshal([]byte(raw), &out); err != nil {
|
||||
return capabilityRouteDecision{}, err
|
||||
}
|
||||
out.Reason = strings.TrimSpace(out.Reason)
|
||||
return out, nil
|
||||
}
|
||||
34
internal/agent/router_parser_test.go
Normal file
34
internal/agent/router_parser_test.go
Normal file
@@ -0,0 +1,34 @@
|
||||
package agent
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseCapabilityRoute(t *testing.T) {
|
||||
raw := `{"need_skills":true,"selected_tools":["shell"],"selected_skills":["文件系统查询专家"],"reason":"需要技能约束"}`
|
||||
out, err := parseCapabilityRoute(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parseCapabilityRoute error: %v", err)
|
||||
}
|
||||
if !out.NeedSkills {
|
||||
t.Fatal("expected need_skills=true")
|
||||
}
|
||||
if len(out.SelectedTools) != 1 || out.SelectedTools[0] != "shell" {
|
||||
t.Fatalf("unexpected selected_tools: %#v", out.SelectedTools)
|
||||
}
|
||||
if len(out.SelectedSkills) != 1 || out.SelectedSkills[0] != "文件系统查询专家" {
|
||||
t.Fatalf("unexpected selected_skills: %#v", out.SelectedSkills)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCapabilityRouteCodeFence(t *testing.T) {
|
||||
raw := "```json\n{\"need_skills\":false,\"selected_tools\":[\"file\",\"shell\"],\"selected_skills\":[],\"reason\":\"工具足够\"}\n```"
|
||||
out, err := parseCapabilityRoute(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parseCapabilityRoute error: %v", err)
|
||||
}
|
||||
if out.NeedSkills {
|
||||
t.Fatal("expected need_skills=false")
|
||||
}
|
||||
if len(out.SelectedTools) != 2 {
|
||||
t.Fatalf("unexpected selected_tools len: %d", len(out.SelectedTools))
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -18,6 +19,20 @@ type Client interface {
|
||||
Generate(ctx context.Context, systemPrompt, userPrompt string) (string, error)
|
||||
}
|
||||
|
||||
type FileChatClient interface {
|
||||
GenerateWithFiles(ctx context.Context, systemPrompt, userPrompt string, fileIDs []string) (string, error)
|
||||
}
|
||||
|
||||
type FileUploader interface {
|
||||
UploadFile(ctx context.Context, file InputFile, purpose string) (string, error)
|
||||
}
|
||||
|
||||
type InputFile struct {
|
||||
FileName string
|
||||
MimeType string
|
||||
Content []byte
|
||||
}
|
||||
|
||||
type OpenAICompatibleClient struct {
|
||||
baseURL string
|
||||
apiKey string
|
||||
@@ -43,27 +58,64 @@ type chatRequest struct {
|
||||
|
||||
type chatMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
Content any `json:"content"`
|
||||
}
|
||||
|
||||
type chatContentPart struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text,omitempty"`
|
||||
FileID string `json:"file_id,omitempty"`
|
||||
}
|
||||
|
||||
type chatResponse struct {
|
||||
Choices []struct {
|
||||
Message chatMessage `json:"message"`
|
||||
Message struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (c *OpenAICompatibleClient) Generate(ctx context.Context, systemPrompt, userPrompt string) (string, error) {
|
||||
if c.log != nil {
|
||||
c.log.Debugf("llm request start model=%s system_len=%d user_len=%d", c.model, len(systemPrompt), len(userPrompt))
|
||||
type fileUploadResponse struct {
|
||||
ID string `json:"id"`
|
||||
Bytes int64 `json:"bytes,omitempty"`
|
||||
CreatedAt int64 `json:"created_at,omitempty"`
|
||||
Filename string `json:"filename,omitempty"`
|
||||
Object string `json:"object,omitempty"`
|
||||
Purpose string `json:"purpose,omitempty"`
|
||||
Code int `json:"code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Status any `json:"status,omitempty"`
|
||||
StatusDetails any `json:"status_details,omitempty"`
|
||||
Data *struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"data,omitempty"`
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (c *OpenAICompatibleClient) Generate(ctx context.Context, systemPrompt, userPrompt string) (string, error) {
|
||||
return c.generateInternal(ctx, systemPrompt, userPrompt, nil)
|
||||
}
|
||||
|
||||
func (c *OpenAICompatibleClient) GenerateWithFiles(ctx context.Context, systemPrompt, userPrompt string, fileIDs []string) (string, error) {
|
||||
return c.generateInternal(ctx, systemPrompt, userPrompt, fileIDs)
|
||||
}
|
||||
|
||||
func (c *OpenAICompatibleClient) generateInternal(ctx context.Context, systemPrompt, userPrompt string, fileIDs []string) (string, error) {
|
||||
if c.log != nil {
|
||||
c.log.Debugf("llm request start model=%s system_len=%d user_len=%d file_count=%d", c.model, len(systemPrompt), len(userPrompt), len(fileIDs))
|
||||
}
|
||||
userContent := buildUserContent(userPrompt, fileIDs)
|
||||
body := chatRequest{
|
||||
Model: c.model,
|
||||
Messages: []chatMessage{
|
||||
{Role: "system", Content: systemPrompt},
|
||||
{Role: "user", Content: userPrompt},
|
||||
{Role: "user", Content: userContent},
|
||||
},
|
||||
}
|
||||
b, err := json.Marshal(body)
|
||||
@@ -132,3 +184,144 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, systemPrompt, use
|
||||
|
||||
return out.Choices[0].Message.Content, nil
|
||||
}
|
||||
|
||||
func buildUserContent(userPrompt string, fileIDs []string) any {
|
||||
trimmedPrompt := strings.TrimSpace(userPrompt)
|
||||
if len(fileIDs) == 0 {
|
||||
return userPrompt
|
||||
}
|
||||
|
||||
parts := make([]chatContentPart, 0, len(fileIDs)+1)
|
||||
if trimmedPrompt != "" {
|
||||
parts = append(parts, chatContentPart{Type: "text", Text: userPrompt})
|
||||
}
|
||||
for _, id := range fileIDs {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
parts = append(parts, chatContentPart{Type: "file", FileID: id})
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return userPrompt
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
func (c *OpenAICompatibleClient) UploadFile(ctx context.Context, file InputFile, purpose string) (string, error) {
|
||||
if strings.TrimSpace(file.FileName) == "" {
|
||||
return "", fmt.Errorf("empty file name")
|
||||
}
|
||||
if len(file.Content) == 0 {
|
||||
return "", fmt.Errorf("empty file content")
|
||||
}
|
||||
purpose = strings.TrimSpace(purpose)
|
||||
purposes := []string{}
|
||||
if purpose != "" {
|
||||
purposes = append(purposes, purpose)
|
||||
}
|
||||
// Provider compatibility fallback order.
|
||||
purposes = appendIfMissing(purposes, "file-extract")
|
||||
purposes = appendIfMissing(purposes, "batch")
|
||||
|
||||
var lastErr error
|
||||
for _, p := range purposes {
|
||||
fileID, err := c.uploadFileOnce(ctx, file, p)
|
||||
if err == nil {
|
||||
return fileID, nil
|
||||
}
|
||||
lastErr = err
|
||||
if c.log != nil {
|
||||
c.log.Warnf("llm file upload failed purpose=%s err=%v", p, err)
|
||||
}
|
||||
}
|
||||
if lastErr == nil {
|
||||
lastErr = fmt.Errorf("llm file upload failed: no purpose tried")
|
||||
}
|
||||
return "", lastErr
|
||||
}
|
||||
|
||||
func (c *OpenAICompatibleClient) uploadFileOnce(ctx context.Context, file InputFile, purpose string) (string, error) {
|
||||
|
||||
body := &bytes.Buffer{}
|
||||
writer := multipart.NewWriter(body)
|
||||
if err := writer.WriteField("purpose", purpose); err != nil {
|
||||
return "", err
|
||||
}
|
||||
part, err := writer.CreateFormFile("file", file.FileName)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := part.Write(file.Content); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
url := strings.TrimRight(c.baseURL, "/") + "/files"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
req.Header.Set("Authorization", "Bearer "+c.apiKey)
|
||||
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
raw, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var out fileUploadResponse
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return "", fmt.Errorf("llm file upload response decode failed: %w body=%s", err, clipForError(raw))
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
if strings.TrimSpace(out.Message) != "" {
|
||||
return "", fmt.Errorf("llm file upload error: %s", out.Message)
|
||||
}
|
||||
if out.Error != nil && out.Error.Message != "" {
|
||||
return "", fmt.Errorf("llm file upload error: %s", out.Error.Message)
|
||||
}
|
||||
return "", fmt.Errorf("llm file upload status: %d body=%s", resp.StatusCode, clipForError(raw))
|
||||
}
|
||||
fileID := strings.TrimSpace(out.ID)
|
||||
if fileID == "" && out.Data != nil {
|
||||
fileID = strings.TrimSpace(out.Data.ID)
|
||||
}
|
||||
if fileID == "" {
|
||||
return "", fmt.Errorf("llm file upload returned empty file id body=%s", clipForError(raw))
|
||||
}
|
||||
if c.log != nil {
|
||||
c.log.Infof("llm file uploaded name=%s size=%d file_id=%s purpose=%s status=%v", file.FileName, len(file.Content), fileID, purpose, out.Status)
|
||||
}
|
||||
return fileID, nil
|
||||
}
|
||||
|
||||
func clipForError(raw []byte) string {
|
||||
s := strings.TrimSpace(string(raw))
|
||||
const max = 400
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
return s[:max] + "...(truncated)"
|
||||
}
|
||||
|
||||
func appendIfMissing(items []string, value string) []string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return items
|
||||
}
|
||||
for _, it := range items {
|
||||
if strings.EqualFold(strings.TrimSpace(it), value) {
|
||||
return items
|
||||
}
|
||||
}
|
||||
return append(items, value)
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"laodingbot/internal/logger"
|
||||
"laodingbot/internal/tools"
|
||||
"laodingbot/tools/fileoperation"
|
||||
"laodingbot/tools/git"
|
||||
"laodingbot/tools/shell"
|
||||
"laodingbot/tools/websearch"
|
||||
)
|
||||
@@ -16,6 +17,7 @@ import (
|
||||
func RunChild(ctx context.Context, cfg config.Config, log *logger.Logger) error {
|
||||
var registryLog *logger.Logger
|
||||
var fileLog *logger.Logger
|
||||
var gitLog *logger.Logger
|
||||
var shellLog *logger.Logger
|
||||
var searchLog *logger.Logger
|
||||
var serverLog *logger.Logger
|
||||
@@ -23,12 +25,19 @@ func RunChild(ctx context.Context, cfg config.Config, log *logger.Logger) error
|
||||
log.Infof("toolhost child starting")
|
||||
registryLog = log.WithComponent("toolhost.registry")
|
||||
fileLog = log.WithComponent("toolhost.file")
|
||||
gitLog = log.WithComponent("toolhost.git")
|
||||
shellLog = log.WithComponent("toolhost.shell")
|
||||
searchLog = log.WithComponent("toolhost.websearch")
|
||||
serverLog = log.WithComponent("toolhost.server")
|
||||
}
|
||||
registry := tools.NewRegistry(registryLog)
|
||||
registry.Register(fileoperation.New(cfg.Security.AllowedDirs, cfg.ToolOutputMaxChars, fileLog))
|
||||
registry.Register(git.New(
|
||||
cfg.Security.WorkDir,
|
||||
time.Duration(cfg.ToolCallTimeoutSec)*time.Second,
|
||||
cfg.ToolOutputMaxChars,
|
||||
gitLog,
|
||||
))
|
||||
registry.Register(shell.New(
|
||||
cfg.Security.AllowedCommands,
|
||||
cfg.Security.WorkDir,
|
||||
|
||||
@@ -4,6 +4,10 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -34,9 +38,17 @@ type IncomingMessage struct {
|
||||
MessageID string
|
||||
ChatID string
|
||||
UserID string
|
||||
MsgType string
|
||||
Text string
|
||||
FileName string
|
||||
FileKey string
|
||||
FileMime string
|
||||
FileBytes []byte
|
||||
FilePath string
|
||||
}
|
||||
|
||||
const maxFeishuFileBytes = 20 * 1024 * 1024
|
||||
|
||||
func NewBot(appID, appSecret, verifyToken, _ string, _ string, log *logger.Logger) (*Bot, error) {
|
||||
if appID == "" || appSecret == "" {
|
||||
return nil, fmt.Errorf("empty feishu app credentials")
|
||||
@@ -66,7 +78,7 @@ func (b *Bot) Run(ctx context.Context, handler func(context.Context, IncomingMes
|
||||
incoming, ok := parseIncoming(event)
|
||||
if !ok {
|
||||
if b.log != nil {
|
||||
b.log.Debugf("skip non-text or invalid feishu event")
|
||||
b.log.Debugf("skip unsupported or invalid feishu event")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -76,8 +88,11 @@ func (b *Bot) Run(ctx context.Context, handler func(context.Context, IncomingMes
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if incoming.MsgType == "file" {
|
||||
b.enrichFileIncoming(evtCtx, &incoming)
|
||||
}
|
||||
if b.log != nil {
|
||||
b.log.Infof("feishu message received message_id=%s chat_id=%s user_id=%s text=%s", incoming.MessageID, incoming.ChatID, incoming.UserID, incoming.Text)
|
||||
b.log.Infof("feishu message received message_id=%s chat_id=%s user_id=%s msg_type=%s text=%s", incoming.MessageID, incoming.ChatID, incoming.UserID, incoming.MsgType, incoming.Text)
|
||||
}
|
||||
reply, err := handler(evtCtx, incoming)
|
||||
if err != nil {
|
||||
@@ -159,6 +174,175 @@ func extractText(content string) (string, error) {
|
||||
return parsed.Text, nil
|
||||
}
|
||||
|
||||
func extractFileMeta(content string) (fileName string, fileKey string, err error) {
|
||||
var parsed map[string]any
|
||||
if err := json.Unmarshal([]byte(content), &parsed); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
readString := func(keys ...string) string {
|
||||
for _, key := range keys {
|
||||
if v, ok := parsed[key]; ok {
|
||||
s, ok := v.(string)
|
||||
if ok {
|
||||
trimmed := strings.TrimSpace(s)
|
||||
if trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
fileName = readString("file_name", "fileName", "name", "filename")
|
||||
fileKey = readString("file_key", "fileKey", "key")
|
||||
return fileName, fileKey, nil
|
||||
}
|
||||
|
||||
func buildFileRecognitionText(fileName, fileKey string) string {
|
||||
if strings.TrimSpace(fileName) == "" {
|
||||
fileName = "(unknown)"
|
||||
}
|
||||
if strings.TrimSpace(fileKey) == "" {
|
||||
fileKey = "(unknown)"
|
||||
}
|
||||
|
||||
return strings.Join([]string{
|
||||
"用户发送了一条飞书文件消息。",
|
||||
"文件名: " + fileName,
|
||||
"文件Key: " + fileKey,
|
||||
"系统将先上传该文件到 LLM Provider，再由模型完成文档解析。若上传失败，本次请求将直接中止。",
|
||||
}, "\n")
|
||||
}
|
||||
|
||||
func (b *Bot) enrichFileIncoming(ctx context.Context, incoming *IncomingMessage) {
|
||||
if incoming == nil {
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(incoming.MessageID) == "" || strings.TrimSpace(incoming.FileKey) == "" {
|
||||
incoming.Text = buildFileRecognitionText(incoming.FileName, incoming.FileKey)
|
||||
incoming.Text += "\n\n未找到完整 file_key 或 message_id，暂时无法下载文件内容。"
|
||||
return
|
||||
}
|
||||
|
||||
content, fileName, err := b.downloadFileContent(ctx, incoming.MessageID, incoming.FileKey)
|
||||
if err != nil {
|
||||
if b.log != nil {
|
||||
b.log.Warnf("feishu download file content failed message_id=%s file_key=%s err=%v", incoming.MessageID, incoming.FileKey, err)
|
||||
}
|
||||
incoming.Text = buildFileRecognitionText(incoming.FileName, incoming.FileKey)
|
||||
incoming.Text += "\n\n文件下载失败: " + err.Error()
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(fileName) != "" {
|
||||
incoming.FileName = fileName
|
||||
}
|
||||
incoming.FileBytes = content
|
||||
incoming.FileMime = detectMimeByName(incoming.FileName)
|
||||
localPath, saveErr := saveIncomingFile("files", incoming.FileName, incoming.FileBytes)
|
||||
if saveErr != nil {
|
||||
if b.log != nil {
|
||||
b.log.Warnf("save incoming feishu file failed name=%s err=%v", incoming.FileName, saveErr)
|
||||
}
|
||||
incoming.Text = buildFileRecognitionText(incoming.FileName, incoming.FileKey)
|
||||
incoming.Text += "\n\n文件已下载但本地保存失败: " + saveErr.Error()
|
||||
return
|
||||
}
|
||||
incoming.FilePath = localPath
|
||||
incoming.Text = buildFileRecognitionText(incoming.FileName, incoming.FileKey)
|
||||
incoming.Text += fmt.Sprintf("\n\n文件已下载并保存到本地，路径=%s，大小=%d bytes，mime=%s。", incoming.FilePath, len(content), incoming.FileMime)
|
||||
}
|
||||
|
||||
func (b *Bot) downloadFileContent(ctx context.Context, messageID, fileKey string) ([]byte, string, error) {
|
||||
req := larkim.NewGetMessageResourceReqBuilder().
|
||||
MessageId(messageID).
|
||||
FileKey(fileKey).
|
||||
Type("file").
|
||||
Build()
|
||||
|
||||
resp, err := b.apiClient.Im.MessageResource.Get(ctx, req)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if resp == nil || resp.File == nil {
|
||||
if resp != nil {
|
||||
return nil, "", fmt.Errorf("empty file stream code=%d msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
return nil, "", fmt.Errorf("empty file stream")
|
||||
}
|
||||
|
||||
bts, err := io.ReadAll(resp.File)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if len(bts) > maxFeishuFileBytes {
|
||||
return nil, "", fmt.Errorf("file too large: %d bytes, max=%d", len(bts), maxFeishuFileBytes)
|
||||
}
|
||||
return bts, strings.TrimSpace(resp.FileName), nil
|
||||
}
|
||||
|
||||
func detectMimeByName(fileName string) string {
|
||||
ext := strings.ToLower(strings.TrimSpace(filepath.Ext(fileName)))
|
||||
if ext == "" {
|
||||
return "application/octet-stream"
|
||||
}
|
||||
m := strings.TrimSpace(mime.TypeByExtension(ext))
|
||||
if m == "" {
|
||||
return "application/octet-stream"
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func saveIncomingFile(baseDir, fileName string, content []byte) (string, error) {
|
||||
if len(content) == 0 {
|
||||
return "", fmt.Errorf("empty file content")
|
||||
}
|
||||
if strings.TrimSpace(baseDir) == "" {
|
||||
baseDir = "files"
|
||||
}
|
||||
if err := os.MkdirAll(baseDir, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
safeName := sanitizeFileName(fileName)
|
||||
if safeName == "" {
|
||||
safeName = "upload.bin"
|
||||
}
|
||||
finalName := fmt.Sprintf("%d_%s", time.Now().UnixNano(), safeName)
|
||||
target := filepath.Join(baseDir, finalName)
|
||||
if err := os.WriteFile(target, content, 0o644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
abs, err := filepath.Abs(target)
|
||||
if err != nil {
|
||||
return target, nil
|
||||
}
|
||||
return abs, nil
|
||||
}
|
||||
|
||||
func sanitizeFileName(fileName string) string {
|
||||
name := strings.TrimSpace(filepath.Base(fileName))
|
||||
if name == "" || name == "." || name == ".." {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range name {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
|
||||
b.WriteRune(r)
|
||||
continue
|
||||
}
|
||||
b.WriteByte('_')
|
||||
}
|
||||
out := strings.TrimSpace(b.String())
|
||||
if out == "" || out == "." || out == ".." {
|
||||
return ""
|
||||
}
|
||||
if strings.HasPrefix(out, ".") {
|
||||
out = "file" + out
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseIncoming(event *larkim.P2MessageReceiveV1) (IncomingMessage, bool) {
|
||||
if event == nil || event.Event == nil || event.Event.Message == nil || event.Event.Sender == nil || event.Event.Sender.SenderId == nil {
|
||||
return IncomingMessage{}, false
|
||||
@@ -168,12 +352,11 @@ func parseIncoming(event *larkim.P2MessageReceiveV1) (IncomingMessage, bool) {
|
||||
}
|
||||
|
||||
msg := event.Event.Message
|
||||
if msg.MessageType == nil || *msg.MessageType != "text" || msg.ChatId == nil || msg.Content == nil || msg.MessageId == nil {
|
||||
if msg.MessageType == nil || msg.ChatId == nil || msg.Content == nil || msg.MessageId == nil {
|
||||
return IncomingMessage{}, false
|
||||
}
|
||||
|
||||
text, err := extractText(*msg.Content)
|
||||
if err != nil {
|
||||
msgType := strings.TrimSpace(*msg.MessageType)
|
||||
if msgType == "" {
|
||||
return IncomingMessage{}, false
|
||||
}
|
||||
|
||||
@@ -186,12 +369,33 @@ func parseIncoming(event *larkim.P2MessageReceiveV1) (IncomingMessage, bool) {
|
||||
userID = *event.Event.Sender.SenderId.UnionId
|
||||
}
|
||||
|
||||
return IncomingMessage{
|
||||
incoming := IncomingMessage{
|
||||
MessageID: *msg.MessageId,
|
||||
ChatID: *msg.ChatId,
|
||||
UserID: userID,
|
||||
Text: text,
|
||||
}, true
|
||||
MsgType: msgType,
|
||||
}
|
||||
|
||||
switch msgType {
|
||||
case "text":
|
||||
text, err := extractText(*msg.Content)
|
||||
if err != nil {
|
||||
return IncomingMessage{}, false
|
||||
}
|
||||
incoming.Text = text
|
||||
return incoming, true
|
||||
case "file":
|
||||
fileName, fileKey, err := extractFileMeta(*msg.Content)
|
||||
if err != nil {
|
||||
return IncomingMessage{}, false
|
||||
}
|
||||
incoming.FileName = fileName
|
||||
incoming.FileKey = fileKey
|
||||
incoming.Text = buildFileRecognitionText(fileName, fileKey)
|
||||
return incoming, true
|
||||
default:
|
||||
return IncomingMessage{}, false
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Bot) shouldProcessMessage(messageID string) bool {
|
||||
|
||||
138
internal/transport/feishu/bot_test.go
Normal file
138
internal/transport/feishu/bot_test.go
Normal file
@@ -0,0 +1,138 @@
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
larkim "github.com/larksuite/oapi-sdk-go/v3/service/im/v1"
|
||||
)
|
||||
|
||||
func mustEventFromJSON(t *testing.T, raw string) *larkim.P2MessageReceiveV1 {
|
||||
t.Helper()
|
||||
var evt larkim.P2MessageReceiveV1
|
||||
if err := json.Unmarshal([]byte(raw), &evt); err != nil {
|
||||
t.Fatalf("unmarshal event json failed: %v", err)
|
||||
}
|
||||
return &evt
|
||||
}
|
||||
|
||||
func TestParseIncomingText(t *testing.T) {
|
||||
evt := mustEventFromJSON(t, `{
|
||||
"event": {
|
||||
"message": {
|
||||
"message_id": "msg_text_1",
|
||||
"chat_id": "chat_1",
|
||||
"message_type": "text",
|
||||
"content": "{\"text\":\"你好\"}"
|
||||
},
|
||||
"sender": {
|
||||
"sender_type": "user",
|
||||
"sender_id": {"open_id": "u_open_1"}
|
||||
}
|
||||
}
|
||||
}`)
|
||||
|
||||
in, ok := parseIncoming(evt)
|
||||
if !ok {
|
||||
t.Fatal("expected text message parse success")
|
||||
}
|
||||
if in.MsgType != "text" {
|
||||
t.Fatalf("expected msg type text, got %s", in.MsgType)
|
||||
}
|
||||
if in.Text != "你好" {
|
||||
t.Fatalf("unexpected text: %q", in.Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseIncomingFile(t *testing.T) {
|
||||
evt := mustEventFromJSON(t, `{
|
||||
"event": {
|
||||
"message": {
|
||||
"message_id": "msg_file_1",
|
||||
"chat_id": "chat_1",
|
||||
"message_type": "file",
|
||||
"content": "{\"file_key\":\"file_key_123\",\"file_name\":\"report.pdf\"}"
|
||||
},
|
||||
"sender": {
|
||||
"sender_type": "user",
|
||||
"sender_id": {"user_id": "u_id_1"}
|
||||
}
|
||||
}
|
||||
}`)
|
||||
|
||||
in, ok := parseIncoming(evt)
|
||||
if !ok {
|
||||
t.Fatal("expected file message parse success")
|
||||
}
|
||||
if in.MsgType != "file" {
|
||||
t.Fatalf("expected msg type file, got %s", in.MsgType)
|
||||
}
|
||||
if in.FileName != "report.pdf" || in.FileKey != "file_key_123" {
|
||||
t.Fatalf("unexpected file meta: name=%q key=%q", in.FileName, in.FileKey)
|
||||
}
|
||||
if !strings.Contains(in.Text, "飞书文件消息") {
|
||||
t.Fatalf("expected synthesized text mentions file message, got: %q", in.Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseIncomingUnsupportedType(t *testing.T) {
|
||||
evt := mustEventFromJSON(t, `{
|
||||
"event": {
|
||||
"message": {
|
||||
"message_id": "msg_image_1",
|
||||
"chat_id": "chat_1",
|
||||
"message_type": "image",
|
||||
"content": "{\"image_key\":\"img_1\"}"
|
||||
},
|
||||
"sender": {
|
||||
"sender_type": "user",
|
||||
"sender_id": {"open_id": "u_open_1"}
|
||||
}
|
||||
}
|
||||
}`)
|
||||
|
||||
_, ok := parseIncoming(evt)
|
||||
if ok {
|
||||
t.Fatal("expected unsupported message type rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectMimeByName(t *testing.T) {
|
||||
if got := detectMimeByName("report.pdf"); !strings.Contains(got, "pdf") {
|
||||
t.Fatalf("expected pdf mime, got: %s", got)
|
||||
}
|
||||
if got := detectMimeByName("unknown.custom"); got != "application/octet-stream" {
|
||||
t.Fatalf("expected octet-stream fallback, got: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveIncomingFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path, err := saveIncomingFile(dir, "report.pdf", []byte("hello"))
|
||||
if err != nil {
|
||||
t.Fatalf("saveIncomingFile error: %v", err)
|
||||
}
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read saved file failed: %v", err)
|
||||
}
|
||||
if string(b) != "hello" {
|
||||
t.Fatalf("unexpected saved content: %q", string(b))
|
||||
}
|
||||
if filepath.Ext(path) != ".pdf" {
|
||||
t.Fatalf("expected .pdf extension, got: %s", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeFileName(t *testing.T) {
|
||||
got := sanitizeFileName("../bad path/测 试?.pdf")
|
||||
if strings.Contains(got, "/") || strings.Contains(got, "\\") {
|
||||
t.Fatalf("expected sanitized basename only, got: %q", got)
|
||||
}
|
||||
if !strings.HasSuffix(got, ".pdf") {
|
||||
t.Fatalf("expected .pdf suffix, got: %q", got)
|
||||
}
|
||||
}
|
||||
29
skills/architecture-wbs-breakdown/skill.md
Normal file
29
skills/architecture-wbs-breakdown/skill.md
Normal file
@@ -0,0 +1,29 @@
|
||||
---
|
||||
name: architecture-wbs-breakdown
|
||||
description: 作为架构师，将复杂需求拆解为多团队 WBS 详细文档，并自动提交至 Git 仓库
|
||||
---
|
||||
|
||||
# 复杂系统需求拆解与多团队 WBS 规划
|
||||
|
||||
## 使用场景
|
||||
当接收到业务需求文档，需要进行工程化拆解并进行版本化归档时使用，例如：
|
||||
|
||||
- 将单体需求拆解为适合多团队并行开发的 SAFe/PI 规划 WBS
|
||||
- 界定系统中不同子系统（或软硬件团队）的交互边界与契约
|
||||
- 将生成的拆解报告自动化落地并推送到远程代码仓库
|
||||
|
||||
## 架构拆分与团队规划
|
||||
- 提取核心业务流程与全局非功能性需求 (NFR)
|
||||
- 运用限界上下文划分独立演进的子系统，明确交互契约（API、RPC、ICD 等）
|
||||
- 为每个子系统配置专属开发团队，定义角色配置（SM, PO, 开发, 测试）
|
||||
|
||||
## WBS 拆解与输出约束
|
||||
- 按 Epic -> Feature -> Story/Task 的敏捷层级，输出结构清晰、详细的 Markdown 文档
|
||||
- 明确跨团队的交付物契约，梳理 Feature 依赖关系以识别集成风险点
|
||||
|
||||
## 文档归档与版本控制 (工作流执行)
|
||||
文档生成完毕后，必须触发以下自动化流程：
|
||||
- **文件保存**：将生成的 Markdown 文档以合适的命名（如 `wbs_规划_[日期].md`）写入本地的 `new_floder` 文件夹中。
|
||||
- **版本追踪**：使用 Git 管理该文件夹，执行 `git add .`（或指定该 md 文件）。
|
||||
- **提交变更**：生成准确的提交信息（例如 `docs: add WBS breakdown for [项目/需求简述]`），执行 `git commit`。
|
||||
- **远程同步**：将本地提交推送到远程 Git 仓库，执行 `git push`。
|
||||
149
tools/git/git.go
Normal file
149
tools/git/git.go
Normal file
@@ -0,0 +1,149 @@
|
||||
package git
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"laodingbot/internal/logger"
|
||||
)
|
||||
|
||||
// Tool provides common Git operations through a safe command whitelist.
|
||||
type Tool struct {
|
||||
repoDir string
|
||||
allowedSubcmds map[string]struct{}
|
||||
timeout time.Duration
|
||||
maxOutputChars int
|
||||
log *logger.Logger
|
||||
}
|
||||
|
||||
// New creates a new git tool.
|
||||
func New(repoDir string, timeout time.Duration, maxOutputChars int, log *logger.Logger) *Tool {
|
||||
absRepo, err := filepath.Abs(strings.TrimSpace(repoDir))
|
||||
if err != nil {
|
||||
absRepo = strings.TrimSpace(repoDir)
|
||||
}
|
||||
if timeout <= 0 {
|
||||
timeout = 20 * time.Second
|
||||
}
|
||||
if maxOutputChars <= 0 {
|
||||
maxOutputChars = 4000
|
||||
}
|
||||
|
||||
allowed := map[string]struct{}{
|
||||
"status": {},
|
||||
"log": {},
|
||||
"show": {},
|
||||
"diff": {},
|
||||
"branch": {},
|
||||
"checkout": {},
|
||||
"switch": {},
|
||||
"restore": {},
|
||||
"add": {},
|
||||
"commit": {},
|
||||
"reset": {},
|
||||
"revert": {},
|
||||
"merge": {},
|
||||
"rebase": {},
|
||||
"cherry-pick": {},
|
||||
"fetch": {},
|
||||
"pull": {},
|
||||
"push": {},
|
||||
"remote": {},
|
||||
"tag": {},
|
||||
"stash": {},
|
||||
"blame": {},
|
||||
"rev-parse": {},
|
||||
}
|
||||
|
||||
if log != nil {
|
||||
log.Infof("git tool initialized repo_dir=%s timeout=%s max_output_chars=%d", absRepo, timeout, maxOutputChars)
|
||||
}
|
||||
|
||||
return &Tool{
|
||||
repoDir: absRepo,
|
||||
allowedSubcmds: allowed,
|
||||
timeout: timeout,
|
||||
maxOutputChars: maxOutputChars,
|
||||
log: log,
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Tool) Name() string { return "git" }
|
||||
|
||||
func (t *Tool) Description() string {
|
||||
return "Run common git commands inside repository. Input examples: status --short | log --oneline -n 10 | add . | commit -m fix | branch"
|
||||
}
|
||||
|
||||
func (t *Tool) Call(ctx context.Context, input string) (string, error) {
|
||||
trimmed := strings.TrimSpace(input)
|
||||
if trimmed == "" {
|
||||
if t.log != nil {
|
||||
t.log.Warnf("git tool rejected empty command")
|
||||
}
|
||||
return "", fmt.Errorf("empty git command")
|
||||
}
|
||||
|
||||
parts := strings.Fields(trimmed)
|
||||
if len(parts) == 0 {
|
||||
return "", fmt.Errorf("empty git command")
|
||||
}
|
||||
|
||||
if strings.EqualFold(parts[0], "git") {
|
||||
parts = parts[1:]
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "", fmt.Errorf("missing git subcommand")
|
||||
}
|
||||
|
||||
subcmd := strings.ToLower(parts[0])
|
||||
if _, ok := t.allowedSubcmds[subcmd]; !ok {
|
||||
if t.log != nil {
|
||||
t.log.Warnf("git tool rejected subcommand=%s", subcmd)
|
||||
}
|
||||
return "", fmt.Errorf("unsupported git subcommand: %s", subcmd)
|
||||
}
|
||||
|
||||
runCtx, cancel := context.WithTimeout(ctx, t.timeout)
|
||||
defer cancel()
|
||||
|
||||
args := append([]string{subcmd}, parts[1:]...)
|
||||
if t.log != nil {
|
||||
t.log.Infof("git command start subcommand=%s args=%d full=%q", subcmd, len(parts)-1, strings.Join(args, " "))
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(runCtx, "git", args...)
|
||||
cmd.Dir = t.repoDir
|
||||
cmd.Env = append(cmd.Environ(),
|
||||
"GIT_TERMINAL_PROMPT=0",
|
||||
"GIT_PAGER=cat",
|
||||
"GIT_EDITOR=true",
|
||||
)
|
||||
|
||||
out, err := cmd.CombinedOutput()
|
||||
outText := strings.TrimSpace(string(out))
|
||||
if len(outText) > t.maxOutputChars {
|
||||
outText = outText[:t.maxOutputChars]
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if t.log != nil {
|
||||
t.log.Errorf("git command failed subcommand=%s err=%v output=%q", subcmd, err, outText)
|
||||
}
|
||||
if outText == "" {
|
||||
return "", err
|
||||
}
|
||||
return outText, err
|
||||
}
|
||||
|
||||
if t.log != nil {
|
||||
t.log.Infof("git command success subcommand=%s output_bytes=%d", subcmd, len(out))
|
||||
}
|
||||
if outText == "" {
|
||||
return "ok", nil
|
||||
}
|
||||
return outText, nil
|
||||
}
|
||||
129
tools/git/git_test.go
Normal file
129
tools/git/git_test.go
Normal file
@@ -0,0 +1,129 @@
|
||||
package git
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCallRejectsEmptyCommand(t *testing.T) {
|
||||
tool := New(t.TempDir(), 3*time.Second, 4000, nil)
|
||||
_, err := tool.Call(context.Background(), " ")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty command")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCallRejectsUnsupportedSubcommand(t *testing.T) {
|
||||
repo := initTestRepo(t)
|
||||
tool := New(repo, 5*time.Second, 4000, nil)
|
||||
|
||||
_, err := tool.Call(context.Background(), "bisect start")
|
||||
if err == nil {
|
||||
t.Fatal("expected unsupported subcommand error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "unsupported git subcommand") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusAndLog(t *testing.T) {
|
||||
repo := initTestRepo(t)
|
||||
tool := New(repo, 5*time.Second, 4000, nil)
|
||||
|
||||
status, err := tool.Call(context.Background(), "status --short")
|
||||
if err != nil {
|
||||
t.Fatalf("status failed: %v", err)
|
||||
}
|
||||
if strings.TrimSpace(status) != "" && status != "ok" {
|
||||
t.Fatalf("expected clean status output, got: %q", status)
|
||||
}
|
||||
|
||||
logOut, err := tool.Call(context.Background(), "git log --oneline -n 1")
|
||||
if err != nil {
|
||||
t.Fatalf("log failed: %v", err)
|
||||
}
|
||||
if !strings.Contains(logOut, "initial commit") {
|
||||
t.Fatalf("expected initial commit in log output, got: %q", logOut)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddCommitAndDiff(t *testing.T) {
|
||||
repo := initTestRepo(t)
|
||||
tool := New(repo, 8*time.Second, 8000, nil)
|
||||
|
||||
file := filepath.Join(repo, "note.txt")
|
||||
if err := os.WriteFile(file, []byte("v1\n"), 0o644); err != nil {
|
||||
t.Fatalf("write file failed: %v", err)
|
||||
}
|
||||
|
||||
_, err := tool.Call(context.Background(), "add note.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("add failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = tool.Call(context.Background(), "commit -m add-note")
|
||||
if err != nil {
|
||||
t.Fatalf("commit failed: %v", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(file, []byte("v2\n"), 0o644); err != nil {
|
||||
t.Fatalf("rewrite file failed: %v", err)
|
||||
}
|
||||
|
||||
diffOut, err := tool.Call(context.Background(), "diff -- note.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("diff failed: %v", err)
|
||||
}
|
||||
if !strings.Contains(diffOut, "-v1") || !strings.Contains(diffOut, "+v2") {
|
||||
t.Fatalf("expected diff output with v1/v2 changes, got: %q", diffOut)
|
||||
}
|
||||
}
|
||||
|
||||
func initTestRepo(t *testing.T) string {
|
||||
t.Helper()
|
||||
if _, err := exec.LookPath("git"); err != nil {
|
||||
t.Skip("git is not installed")
|
||||
}
|
||||
|
||||
repo := t.TempDir()
|
||||
runGit(t, repo, "init")
|
||||
runGit(t, repo, "config", "user.name", "test-user")
|
||||
runGit(t, repo, "config", "user.email", "test@example.com")
|
||||
|
||||
readme := filepath.Join(repo, "README.md")
|
||||
if err := os.WriteFile(readme, []byte("hello\n"), 0o644); err != nil {
|
||||
t.Fatalf("write readme failed: %v", err)
|
||||
}
|
||||
runGit(t, repo, "add", "README.md")
|
||||
runGit(t, repo, "commit", "-m", "initial commit")
|
||||
|
||||
return repo
|
||||
}
|
||||
|
||||
func runGit(t *testing.T, repo string, args ...string) {
|
||||
t.Helper()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(ctx, "git", args...)
|
||||
cmd.Dir = repo
|
||||
cmd.Env = append(cmd.Environ(),
|
||||
"GIT_TERMINAL_PROMPT=0",
|
||||
"GIT_PAGER=cat",
|
||||
"GIT_EDITOR=true",
|
||||
)
|
||||
if runtime.GOOS == "windows" {
|
||||
// Keep default behavior; this branch documents cross-platform intent.
|
||||
}
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %s failed: %v\n%s", strings.Join(args, " "), err, string(out))
|
||||
}
|
||||
}
|
||||
@@ -94,6 +94,9 @@ func (t *Tool) Call(ctx context.Context, input string) (string, error) {
|
||||
if runtime.GOOS == "windows" {
|
||||
// Windows 下使用 cmd /C 执行，兼容 date、dir 等内建命令。
|
||||
cmd = exec.CommandContext(runCtx, "cmd", "/C", trimmed)
|
||||
} else if requiresShellParsing(trimmed) {
|
||||
// 包含管道、重定向等语法时，必须交给 shell 解释。
|
||||
cmd = exec.CommandContext(runCtx, "sh", "-c", trimmed)
|
||||
} else {
|
||||
cmd = exec.CommandContext(runCtx, base, parts[1:]...)
|
||||
}
|
||||
@@ -126,3 +129,7 @@ func normalizeWindowsCommand(command string) string {
|
||||
return command
|
||||
}
|
||||
}
|
||||
|
||||
func requiresShellParsing(command string) bool {
|
||||
return strings.ContainsAny(command, "|&;<>()$`\\\n")
|
||||
}
|
||||
|
||||
@@ -40,3 +40,27 @@ func TestCallWindowsDateIsNonInteractive(t *testing.T) {
|
||||
t.Fatal("expected non-empty output for date command")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequiresShellParsing(t *testing.T) {
|
||||
if !requiresShellParsing("echo hi | cat") {
|
||||
t.Fatal("expected pipe command to require shell parsing")
|
||||
}
|
||||
if requiresShellParsing("echo hello") {
|
||||
t.Fatal("expected simple command to not require shell parsing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCallSupportsPipeOnUnix(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("unix-only behavior test")
|
||||
}
|
||||
tool := New([]string{"echo"}, ".", 3*time.Second, 4000, nil)
|
||||
out, err := tool.Call(context.Background(), "printf hello | wc -c")
|
||||
if err != nil {
|
||||
t.Fatalf("expected pipeline command success, got err=%v output=%q", err, out)
|
||||
}
|
||||
trimmed := strings.TrimSpace(out)
|
||||
if trimmed != "5" {
|
||||
t.Fatalf("expected output 5, got %q", trimmed)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user