diff --git a/.vscode/launch.json b/.vscode/launch.json index c95919f..c995be1 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -6,7 +6,6 @@ "type": "go", "request": "launch", "mode": "auto", - "preLaunchTask": "Kill Stale LaodingBot Debug Processes", "program": "${workspaceFolder}/cmd/bot", "cwd": "${workspaceFolder}", "envFile": "${workspaceFolder}/configs/env" @@ -16,7 +15,6 @@ "type": "go", "request": "launch", "mode": "auto", - "preLaunchTask": "Kill Stale LaodingBot Debug Processes", "program": "${workspaceFolder}/cmd/bot", "cwd": "${workspaceFolder}", "envFile": "${workspaceFolder}/configs/env", @@ -29,7 +27,6 @@ "type": "go", "request": "launch", "mode": "auto", - "preLaunchTask": "Kill Stale LaodingBot Debug Processes", "program": "${workspaceFolder}/cmd/bot", "cwd": "${workspaceFolder}", "envFile": "${workspaceFolder}/configs/env", diff --git a/.vscode/tasks.json b/.vscode/tasks.json deleted file mode 100644 index 485c2e9..0000000 --- a/.vscode/tasks.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "version": "2.0.0", - "tasks": [ - { - "label": "Kill Stale LaodingBot Debug Processes", - "type": "shell", - "command": "pkill -f '/LaodingBot/cmd/bot/__debug_bin|dlv.*LaodingBot/cmd/bot|dlv dap' || true", - "problemMatcher": [] - } - ] -} diff --git a/cmd/bot/main.go b/cmd/bot/main.go index 18149a1..8d4f72f 100644 --- a/cmd/bot/main.go +++ b/cmd/bot/main.go @@ -207,6 +207,9 @@ func runMessageChannel(ctx context.Context, cfg config.Config, engine *agent.Orc return wb.Run( ctx, func(ctx context.Context, msg webui.IncomingMessage) (string, error) { + if len(msg.FileIDs) > 0 { + return engine.HandleMessageWithFileIDs(ctx, msg.ChatID, msg.UserID, msg.Text, msg.FileIDs) + } return engine.HandleMessage(ctx, msg.ChatID, msg.UserID, msg.Text) }, func(ctx context.Context, chatID, userID string, files []llm.InputFile) ([]string, error) { diff --git a/doc/WebUI开发技术与需求说明.md b/doc/WebUI开发技术与需求说明.md new file mode 100644 index 0000000..26c19ad --- /dev/null +++ b/doc/WebUI开发技术与需求说明.md @@ -0,0 +1,305 @@ +# LaodingBot WebUI 开发技术与需求说明 + +> 版本: v1.0 +> 目标读者: 下一个执行 WebUI 开发任务的 LLM / 前端工程师 +> 依据: 当前后端已落地代码（`internal/transport/webui/bot.go`、`cmd/bot/main.go`、`internal/agent/orchestrator.go`） + +--- + +## 1. 文档目标与范围 + +本说明用于指导“仅 WebUI 页面层/前端层”的开发，确保严格对接当前 LaodingBot 已开放的 WebUI 后端能力。 + +本期范围: +- 支持纯文本对话 +- 支持独立文档上传（上传后立即返回 `file_id`） +- 支持“先上传文档，再发送文本提问”的两阶段交互 + +非本期范围: +- 不改动后端接口语义 +- 不引入新的后端路由 +- 不实现复杂权限系统（当前后端无鉴权） +- 不实现流式输出（当前为一次性响应） + +--- + +## 2. 当前后端能力总览 + +### 2.1 消息通道 +后端已支持 `MESSAGE_CHANNEL=webui`，入口在 `cmd/bot/main.go`。 + +当通道为 `webui` 时: +- 启动 HTTP 服务 +- 暴露两个接口: + - `POST /api/upload` + - `POST /api/chat` + +### 2.2 运行配置 +关键环境变量（`configs/env.sample`）: +- `MESSAGE_CHANNEL=webui` +- `WEBUI_LISTEN_ADDR=:8090` +- `WEBUI_MAX_UPLOAD_MB=20` +- `LLM_*` 相关配置（含文件模型切换） + +### 2.3 文件上下文机制（非常关键） +当前后端使用 `chat_id + user_id` 作为会话键缓存 `file_id`: +- 上传接口会把文件上传到 LLM，并把 `file_id` 缓存到该键 +- 后续 `chat` 接口只要带同一组 `session_id`/`user_id`，就会自动消费缓存文件上下文 + +结论: +- 前端必须稳定维护并复用 `session_id` 与 `user_id` +- 两者变化会导致上传文件上下文丢失（从后端视角是新会话） + +--- + +## 3. 后端 API 契约（以代码为准） + +## 3.1 `POST /api/upload` + +用途: +- 独立上传文档 +- 后端立即调用 LLM 文件上传并返回 `file_id` +- 同时缓存到当前会话，供下一次聊天使用 + +请求: +- Method: `POST` +- Content-Type: `multipart/form-data` +- Form 字段: + - `file` (required) + - `session_id` (optional，建议前端始终传) + - `user_id` (optional，建议前端始终传) + +响应 `200`: +```json +{ + "file_id": "file-xxx", + "file_ids": ["file-xxx"], + "file_name": "report.pdf", + "mime_type": "application/pdf", + "size_bytes": 12345, + "session_id": "sess_xxx", + "user_id": "user_xxx" +} +``` + +错误响应: +- `400`: + - `{"error":"file is required"}` + - `{"error":"invalid multipart form"}` + - `{"error":"invalid file name"}` + - `{"error":"empty file"}` + - `{"error":"read file failed"}` +- `413`: + - `{"error":"file too large"}` +- `500`: + - `{"error":"upload failed"}` + - `{"error":"upload succeeded but file_id is empty"}` + +--- + +## 3.2 `POST /api/chat` + +用途: +- 发送纯文本消息 +- 若该 `session_id + user_id` 下已有上传文件，会自动携带文件上下文进行问答 + +请求: +- Method: `POST` +- Content-Type: `application/json` +- Body: +```json +{ + "text": "请总结我上传的文档", + "session_id": "sess_xxx", + "user_id": "user_xxx" +} +``` + +响应 `200`: +```json +{ + "reply": "...", + "session_id": "sess_xxx", + "user_id": "user_xxx" +} +``` + +错误响应: +- `400`: + - `{"error":"content-type must be application/json"}` + - `{"error":"invalid json body"}` + - `{"error":"text is required"}` +- `405`: + - `{"error":"method not allowed"}` +- `500`: + - `{"error":"chat failed"}` + +--- + +## 4. WebUI 功能需求（给前端实现） + +### 4.1 页面最小能力 +必须实现: +1. 聊天消息列表 +2. 文本输入框 + 发送按钮 +3. 文件选择器 + 上传按钮 +4. 当前会话标识展示（`session_id`、`user_id`） +5. 上传结果区（展示 `file_id`、文件名、大小、时间） +6. 错误提示区（接口失败、文件超限、网络错误） + +### 4.2 会话策略 +必须: +- 首次进入页面时生成并持久化 `session_id`、`user_id`（例如 LocalStorage） +- 每次请求都携带这两个字段 +- 提供“重置会话”操作（清空本地 session/user 并重新生成） + +建议 ID 规则: +- `session_id`: `sess_` +- `user_id`: `user_` + +### 4.3 文件上传交互 +必须: +- 文件选择后显示基本信息（名称、大小、类型） +- 点击上传后调用 `/api/upload` +- 上传成功后将返回的 `file_id` 加入“已上传文件列表” +- 上传失败时给出明确错误提示 + +建议: +- 前端先做大小校验（默认 20MB，可配置） +- 上传中禁用重复点击 + +### 4.4 聊天交互 +必须: +- 发送前校验 `text` 非空 +- 调用 `/api/chat` +- 成功后追加机器人消息 +- 失败后展示错误并保留用户输入 + +建议: +- 支持回车发送（Shift+Enter 换行） +- 增加“请求中”状态，避免并发多发 + +--- + +## 5. 前端状态模型（建议） + +```ts +type UploadedFile = { + fileId: string; + fileName: string; + mimeType: string; + sizeBytes: number; + uploadedAt: number; +}; + +type ChatMessage = { + id: string; + role: "user" | "assistant" | "system"; + content: string; + createdAt: number; + status?: "sending" | "sent" | "failed"; +}; + +type AppState = { + sessionId: string; + userId: string; + messages: ChatMessage[]; + uploadedFiles: UploadedFile[]; + uploading: boolean; + chatting: boolean; + lastError?: string; +}; +``` + +状态约束: +- `uploading=true` 时不应再次触发上传 +- `chatting=true` 时可选禁用发送（或允许排队） +- 任何失败都要更新 `lastError` + +--- + +## 6. 接口调用示例 + +### 6.1 上传 +```bash +curl -X POST http://localhost:8090/api/upload \ + -F "session_id=sess_demo" \ + -F "user_id=user_demo" \ + -F "file=@./demo.pdf" +``` + +### 6.2 聊天 +```bash +curl -X POST http://localhost:8090/api/chat \ + -H "Content-Type: application/json" \ + -d '{"text":"这份文档主要讲了什么？","session_id":"sess_demo","user_id":"user_demo"}' +``` + +--- + +## 7. 开发任务拆解（给下一个 LLM） + +1. 初始化 WebUI 工程（技术栈自选，建议 React + TypeScript）。 +2. 实现全局状态管理（至少管理 session/user/messages/uploads）。 +3. 实现 API Client 层: + - `uploadFile(formData)` + - `sendChat(payload)` + - 统一错误解析 +4. 实现页面组件: + - `ChatPanel` + - `Composer` + - `UploadPanel` + - `SessionBadge` +5. 实现本地持久化: + - 启动时恢复 `session_id`、`user_id` + - 重置会话功能 +6. 完成联调与自测。 + +--- + +## 8. 验收标准 + +必须全部满足: +1. 纯文本聊天可用（无文件也能正常回复）。 +2. 文档上传可用，返回并展示 `file_id`。 +3. 上传后同会话提问能基于文档回答。 +4. 刷新页面后会话 ID 不丢失（可继续提问）。 +5. 文件超限、空消息、网络失败均有可见错误反馈。 +6. 同一会话多文件上传后，后续问答仍可使用文件上下文。 + +--- + +## 9. 已知限制与后续建议 + +当前后端限制（前端需知晓）: +- 无鉴权机制 +- 无流式输出 +- 上传 MIME 未做白名单限制 +- 无会话文件列表接口（前端只能依赖本地记录） + +后续建议（非本期必做）: +1. 新增后端 `GET /api/session/files` 以便页面恢复历史上传列表。 +2. 增加鉴权中间件（Bearer Token/JWT）。 +3. 增加流式聊天接口（SSE/WebSocket）。 +4. 增加上传文件类型白名单和病毒扫描。 + +--- + +## 10. 对下一个 LLM 的执行指令模板 + +可直接复制给下一个 LLM: + +```text +请基于 `doc/WebUI开发技术与需求说明.md` 实现 WebUI 前端，严格对接现有后端接口： +- POST /api/upload +- POST /api/chat + +必须满足： +1) 纯文本聊天 +2) 独立文档上传 +3) 上传后同会话提问可利用文档上下文 +4) session_id/user_id 持久化 +5) 完整错误处理和可视化反馈 + +不允许修改后端接口语义；若需新增接口，请先输出变更提案，不直接改。 +``` diff --git a/files/1772978682808419264_EIP_RequirementsSpecificationGLA__V2-5.pdf b/files/1772978682808419264_EIP_RequirementsSpecificationGLA__V2-5.pdf deleted file mode 100644 index a5f6f50..0000000 Binary files a/files/1772978682808419264_EIP_RequirementsSpecificationGLA__V2-5.pdf and /dev/null differ diff --git a/files/1772979029589132982_EIP_RequirementsSpecificationGLA__V2-5.pdf b/files/1772979029589132982_EIP_RequirementsSpecificationGLA__V2-5.pdf deleted file mode 100644 index a5f6f50..0000000 Binary files a/files/1772979029589132982_EIP_RequirementsSpecificationGLA__V2-5.pdf and /dev/null differ diff --git a/files/1772979118236039688_EIP_RequirementsSpecificationGLA__V2-5.pdf b/files/1772979118236039688_EIP_RequirementsSpecificationGLA__V2-5.pdf deleted file mode 100644 index a5f6f50..0000000 Binary files a/files/1772979118236039688_EIP_RequirementsSpecificationGLA__V2-5.pdf and /dev/null differ diff --git a/files/1772979372720489345_EIP_RequirementsSpecificationGLA__V2-5.pdf b/files/1772979372720489345_EIP_RequirementsSpecificationGLA__V2-5.pdf deleted file mode 100644 index a5f6f50..0000000 Binary files a/files/1772979372720489345_EIP_RequirementsSpecificationGLA__V2-5.pdf and /dev/null differ diff --git a/files/1772979941413990286_EIP_RequirementsSpecificationGLA__V2-5.pdf b/files/1772979941413990286_EIP_RequirementsSpecificationGLA__V2-5.pdf deleted file mode 100644 index a5f6f50..0000000 Binary files a/files/1772979941413990286_EIP_RequirementsSpecificationGLA__V2-5.pdf and /dev/null differ diff --git a/files/1772980103865061914_EIP_RequirementsSpecificationGLA__V2-5.pdf b/files/1772980103865061914_EIP_RequirementsSpecificationGLA__V2-5.pdf deleted file mode 100644 index a5f6f50..0000000 Binary files a/files/1772980103865061914_EIP_RequirementsSpecificationGLA__V2-5.pdf and /dev/null differ diff --git a/files/EIP_RequirementsSpecificationGLA__V2-5.txt b/files/EIP_RequirementsSpecificationGLA__V2-5.txt deleted file mode 100644 index 80f9fc2..0000000 --- a/files/EIP_RequirementsSpecificationGLA__V2-5.txt +++ /dev/null @@ -1,2455 +0,0 @@ -Software Requirements Specification for Urban Platforms (EIP Project) Page i - - - - - Requirements Specification - For - - Urban Platforms - Version 2.2 - (Consultation Stage) - - - - - Prepared by Demand-Side Engagement Stream - - - - -European Innovation Partnership for Smart Cities & Communities (EIP_SCC) -Integrated infrastructure Action Cluster – Urban Platform - Software Requirements Specification for Urban Platforms (EIP Project) Page ii - - - - -Requirements Specification Life-Cycle -Revision history - - Author Date Reason for change Phase Revision - - Dr. Larissa Romualdo-Suzuki 4/12/2015 - Initial Draft Version 0 - Outline - - Dr. Larissa Romualdo-Suzuki 21/12/2015 Working update Core Demand- Version 1 - Side Team - Consultation - - Dr. Larissa Romualdo-Suzuki 05/01/2016 Incorporate cities Development Version 2.1 - comments - - Dr. Larissa Romualdo-Suzuki 29/01/2016 Incorporate suggestions Development Version 2.2 - from Urban Platforms - Workshop held in Brussels - on the 19/01/16. - - - - -Requirements Specification sign-off: - - Name: Andrew Collinge - Role: Demand Side Engagement Lead Date - Organisation: Greater London Authority 11 January 2016 - - - - -Acknowledgement: - -This document has been developed in consultation with a representative group of European cities from -different locations, size, and level of development. Cities include: London, Amsterdam, Barcelona, -Syracuse, Berlin, Gent, Valencia, Murcia, Derry, Copenhagen, Scottish cities, Porto, Riga. We wish to -recognize the commitment, support and ongoing contribution from these cities to develop a common -statement of needs. - Software Requirements Specification for Urban Platforms (EIP Project) Page iii - - - - -Table of Contents -Table of Contents ...................................................................................................................... iii -1. Introduction ............................................................................................................................1 - 1.1 Purpose and Scope of this Document.................................................................................. 1 - 1.2 Context ..................................................................................................................................... 1 - 1.3 Intended Audience .................................................................................................................. 2 - 1.4 Definitions, Standards, and Framework ............................................................................... 3 - 1.5 How to Use this Document .................................................................................................... 2 - 1.6 Product Scope and Perspective ............................................................................................ 5 - 1.7 The Urban Platform Development Stack ............................................................................. 7 - 1.8 User Classes, Characteristics, and User Access ............................................................... 8 - 1.9 User Documentation ............................................................................................................. 10 - 1.10 Design and Implementation Constraints ............................................................................ 10 - 1.11 Assumptions, Alignment with other Action Clusters and Policies ................................... 10 -2. Urban Platform Value Proposition, Use Cases and Functional Requirements ..12 - 2.1 From Value Proposition to Platform Specifications .......................................................... 12 - 2.2 Overall Goal: “City data is exploited to its full potential” .................................................. 13 - 2.3 SUB-GOAL 1: City data is collected in an intelligent manner ......................................... 15 - 2.4 SUB-GOAL 2: City data is managed in a safe and intelligent manner .......................... 20 - 2.5 SUB-GOAL 3: City data is orchestrated in a marketplace ............................................... 23 - 2.6 SUB-GOAL 4: City data is offered in an accessible manner ........................................... 28 - 2.7 SUB-GOAL 5: User’s experience is enhanced by the provision of value-added - services............................................................................................................................................... 31 -3. Non-functional Requirements..........................................................................................34 - 3.1 Run-time Quality Requirements .......................................................................................... 34 - 3.2 Non Run-time Quality Requirements.................................................................................. 37 - 3.3 List of Non-Functional Requirements ................................................................................. 38 -4. Other Requirements ...........................................................................................................39 - 4.1 Minimal Descriptive Metadata Required from Data Provider .......................................... 39 - 4.2 Urban Platform Supported Data Formats .......................................................................... 40 -5. Conclusion & Forward Plans ...........................................................................................40 -Appendix ......................................................................................... Error! Bookmark not defined. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 1 - - - - -1. Introduction -1.1 Purpose and Scope of this Document -The purpose of this document is to provide a common core set of city-needs-led requirements, co- -developed and validated by EU cities, to support the acquisition of “Urban Platforms” by EU cities. -The requirements are made openly available. The ambition is that adoption of these requirements -by EU cities will lead to reduced pre-procurement times, increased confidence in platform designs, -greater levels of collaboration (particularly amongst smaller cities), innovation in business models, -more affordable solutions, and a more secure basis for industry to apply its innovations. This will -lead to accelerated adoption of urban platforms by EU cities, so that they can exploit the potential -of the growing volumes of city data, and improve the services to and outcomes for their residents, -business community and visitors. - -This document provides a primary input to the EIP_SCC Urban Platform Supply-Side and -Standardisation workstreams as a reference for the development of further technical and -operational materials. The requirements illustrate the purpose and complete declaration for the -development of system, as well as system constraints, interface and interactions with other -external applications. - -The scope of this document is limited to common services data platform requirements, and issues -related to the management of city data within an Urban Platform. This considers the full life-cycle -of city data, including: the maintenance of the inventory of existing city data; development of -functional requirements for common services data platform from a city-needs perspective; -identification of metadata and format standards for city data; and identification of policy issues -related to the data platform. It includes in the scope the functionality of software and systems used -to handle existing data catalogues (e.g., open data portals). - -Commercial off-the-shelf (COTS) software or hardware, as well as databases, which manage data -that are not part of Urban Platform’s collecting responsibility (e.g., mobile applications, proprietary -Internet of Things, proprietary data catalogues) are outside the scope of these requirements. - - -1.2 Context -The initial EIP_SCC Demand-Side survey1 clearly indicated that the vast majority of EU cities did -not have an urban platform; that there were a number of significant barriers to adoption (including -funding, capabilities, and the willingness to work across service ‘silos’); that there was limited -collaboration in this area; and as a result most cities were ‘sitting on the fence’. The opportunity -clearly is to unblock this. Notably because urban platforms provide a vital foundation for smart city -infrastructure and service improvements. So their adoption is considered a strategic priority by the -EIP_SCC, and by a growing number of cities. - -There are a number of clear trends that any city, and thus these requirements, must recognise: - • The astounding increase in volumes of city data, driven notably by IoT/sensor/M2M - implementations, and social media - • The pressure to improve and make open data from public (and private) sources; - -1 - Survey performed Q1’15 and reported on the EIP_SCC Marketplace site - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 2 - - - • The continued reality of austerity that drives cities towards transformative solutions - -Cities presently typically have a mix and variety of qualities of legacy systems, with various -degrees of technical ‘platform or platforms’ in place. It is recognised that the roadmap a city might -follow to transition to a more aligned and functional state will differ in many ways; and that the end -point does not envisage ‘one (physical) platform’. However cities will benefit from adopting more -consistent conceptual, and logical architectures, which will lead to more efficient and effective -physical operations, and greater benefits in terms of exploiting city data. - -Furthermore it must be recognised that cities will continue to wrestle with capacity challenges to -improve quality, availability, governance, and appropriate monetisation of city data. - - -1.3 Intended Audience -This document is co-developed by and initially intended for the growing number of city members of -the Demand-Side Engagement Stream of the EIP_SCC Urban Platform initiative. Its formal release -as a common statement of requirements by representative EU cities will form the basis for -commitment by EU cities to the initiative by way of a Letter of Intent (LoI). The spirit of which seeks -to ensure broad use of these requirements by EU cities (and recognition of deviations) to help -open the smart city market. - -It is also intended to be used by Industry in setting an agreed city-needs-led set of requirements -which can become more consistent and common across EU cities. This will lead to advantage for -cities and Industry. - - -1.4 How to Use this Document -Cities are encouraged to use this document as a guiding reference for the development of their -own platform specifications. - -The following steps offer a logical means to extract the full value of this document: - - - 1. Identify the priority service outcomes that the city seeks to achieve to steer the overall - prioritisation of urban platform development - 2. Perform a city data mapping exercise to develop a picture of the data landscape, including: - sources, volume, variety, temporal factors and sensitivity, data licensing and ownership - policies. - 3. Map out existing ICT system resources across the city in order to identify those resources - with the greatest potential for reuse, identify gaps and provide the foundation for a data - strategy and technology plan to fill them. - 4. Identify the requirements within this document that best suit the city needs - alongside any - other emerging requirements – and refine and prioritise them in accordance to their data - strategy defined in step 1 and 2. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 3 - - - 5. Cities can create their own specification document in their own language and refer to the - requirements of this document by using the unique identifier of the requirements (which - signals to providers the similarities and deviations from the core set) - 6. Determine what ICT infrastructure and sets of requirements are needed citywide to support - the urban platform. - - - - -1.5 Definitions, Standards, and Framework -To ensure a robust basis for market adoption we have set in place a number of foundations. Firstly, -a common definition of terms. Secondly, the adoption of some core accepted standards. Thirdly, -an underlying model to develop requirements. - -1.5.1 Working Definitions -For the purpose of this document we assume the working definitions developed within and shared -across all working groups of the Urban Platform initiative - -• An ‘Urban Platform’… - - o Implements a logical architecture/content/design that brings together (integrates) data flows - within and across city systems and exploits modern technologies (sensors, cloud services, - mobile devices, analytics, social media etc); - o Provides the building blocks that enable cities to rapidly shift from fragmented operations to - include predictive effective operations, and novel ways of engaging and serving city - stakeholders; - o Transforms, in a way that is tangible and measurable, outcomes at local level (e.g. increase - energy efficiency, reduce traffic congestion and emissions, create (digital) innovation - ecosystems, efficient city operations for administrations and services). - -• ‘City Data’ is… - - o Data that is held by any organisation - government, public sector, private sector or not-for- - profit - which is providing a service or utility, or is occupying part of the city in a way that can - be said to have a bearing on local populations and the functioning of that place; - o Consists of varied characteristics such as static, near-real time or in the future, real time, - descriptive or operational. - o It will be to a greater extent generated by individual citizens and this too (with due - consideration to privacy and a strong trust framework) can be considered city data. - - -For the purpose of this report, the following terms and definitions4 apply. - - o Open data: non-privacy-restricted and non-confidential data. Produced with either public or - private resource and is made available without any restrictions on its usage or distribution. - o Private data: restricted and/or licensed data including permission, charging, privacy, - publication and distribution. Produced with either public or private resource. - o Commercial data: restricted and/or licensed data including permission, charging, privacy, - publication and distribution. Produced with either public or private resource. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 4 - - - o Sensory data: open and/or restricted data collected by different sensors, actuators and - devices owned by public and private sector, and citizens. Sensory data is usually diverse in - nature and architectural features, mostly location and time dependent, and present different - levels of quality. - -These types of city data are produced within the public and private sectors and crowdsourcing -initiatives, and have their respective suppliers and distribution strategies (e.g. reports, APIs, -datasets). - - o Public sector data: restricted data relating to location, national security, commercial - sensitivity and privacy. Data produced, collected or funded by the public sector. - o Private sector data: data produced, collected or funded by the private sector, which can be - open, private and commercial data. In case of private data, the restrictions of usage and - distribution are decided by the individual businesses. - o Crowd-sourced data: data provided, collected and distributed by humans through the use - of digital technologies and social media. - - -1.5.2 Use of Standards - -• We seek to ensure Urban Platforms in EU cities are designed with regard to international best - practices for data repositories. -• As new guidance and standards are developed the three streams of the EIP_SCC2 Urban - Platform initiative will review them for broader use. (Note: the SCC03 EC project and - consortium is explicitly linked within this initiative and will act on Urban Platform standards). -• For now, this document assumes a modified IEEE 830-19983 layout for software requirements - specification, as the basis to capture a high-level statement of the urban platform’s - requirements. - - -1.5.3 The Organising Framework - -This document uses the dynamic business models framework4 illustrated in Figure 1 to organise -and guide the definition of goals, the development of use cases, and definition of requirements -during the life cycle of urban platforms. Future iterations of this dynamic design process are -Procurement, Roll-Out and Market phases. Consequently, this document may include, alter, -prioritize, extend or remove specifications as they become known. - -The principal goal addressed is that of “exploiting the value of city data through urban -platforms”. From this we have developed five common sub-goals. Each includes: a description; -rationale; drivers; and actions. These sub-goals are elaborated through a total of 14 use cases and -developed into a total of 116 functional and non-functional requirements as illustrated in Figure 2. -The use of the requirements by Industry will be subject to a further update. - - - - -2 - EIP Integrated infrastructure Urban Platform Governance Model, Version 3, 28th July 2015. -3 - IEEE 830-1998 Standard for Software Requirements Specifications, 1998. -4 - Framework for Data Infrastructures Design from “Data as Infrastructure for Smart Cities”, Suzuki-LCSR, -PhD Thesis, UCL, 2015. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 5 - - - - - Figure 1. Organising Framework4. - - - - - Sub-Goal 1: City data is provided - Use Cases: 3 Requirements: 34 - in a harmonised way - value of city data through an - Principle Goal: Exploit the - - - - - Sub-Goal 2: City data is managed - Use Cases: 3 Requirements: 36 - in a safe and intelligent manner - urban platform - - - - - Sub-Goal 3: City data is - Use Cases: 2 Requirements: 12 - orchestrated in a market place - - - - - Sub-Goal 4: City data is offered in - Use Cases: 3 Requirements: 11 - an accessible manner - - - - - Sub-Goal 5: User’s experience is - enhanced by the provision of Use Cases: 3 Requirements: 8 - value-added services - - - - Figure 2 Structure of this document. - - - - -1.6 Product Scope and Perspective -The EIP Project’s urban platform is an open common architecture which serves for city data -collection, management and distribution. An urban platform is intended to support the widespread -exploitation of city data by humans and machines in the urban environment. Figure 3 illustrates a -holistic high level overview of the urban platform the EIP project intends to deliver. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 6 - - - -The reference architecture for urban platforms should: - -• Cater for interoperability between urban infrastructures -• Enable replicability of the solutions/platforms city to city -• Scale without technical constraints and excessive cost increase -• Provide open APIs and SDKs -• Enable Real Time capabilities -• Support implementation of functional and technical capabilities - - - - - Figure 3. High level overview of the urban platform (currently approved EC DG CNECT). - - -The current urban platform market is nascent. Many software vendors offer such a platform, though -many requirements and expectations of the stakeholders of city data are not (fully) addressed. As -a result, current urban platforms are often more costly to design and maintain, less reusable and -often not interoperable platform-to-platform, and susceptible to information fragmentation and -overload. - -The urban platform which the EIP initiative intends to design takes a step beyond the platforms -currently on the market by ensuring the requirements are fully founded on a co-created and -common set of representative city needs, from which it solicits suitable industry input, and an open -and managed collaboration between industry, cities and communities, and others, in order to take -into account their needs and concerns. To do this, it is necessary to take a technology agnostic -approach to design an open and common reference architecture for urban platforms. This platform -must ensure data is collected and sustained in accordance with well-stablished standards, -managed in a robust manner so that it can handle high level supply and demand of data, and -distributed across different value chains, systems and stakeholders. The ability to handle high level -of city data supply and demand while being user secure and accessible enough for city-wide -exploitation of data is one of many keys to the success of urban platforms. This is central to the -design and implementation of urban platforms. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 7 - - -1.7 The Urban Platform Development Stack -The requirements specification of this document is based on the Urban Platform Development -Stack illustrated in Figure 4. The stack is composed by five domains (represented as layers in the -stack) necessary to fully implement an urban platform software suite as shown in Table 1. Each -domain comprehends a set of requirements necessary to the design of a common and open urban -data service platform. The elicited requirements are used to define a technical architecture which is -simple enough to be comprehensible at least at a high level of abstraction. The platform should be -conceptually decomposable into its major subsystems, the platform’s functionality reused by many -services and external applications should be identifiable, and interactions between the platform -and services, data providers and data consumers should be well defined and explicit. - -The first layer of the stack “Societal needs” concerns to outcomes we strive for within a portfolio of -city service domains. An urban platform should recognise societal needs and wants as the starting -point for city data service offering. Ultimately, an urban platform aims to provide tailor made and -compelling engaging services for the users. The Services and Business models layers concerns -with delivering data services which carefully targets the needs and expectations of the different -users of the urban platform, and explore use cases and commercial models where data is used to -deliver different forms of value. The city data layer concerns with the mechanisms necessary to -transform urban platforms into a foundation for widespread exploitation of data, including handling -data architectural features, data usability, semantics and quality aspects. The urban platform layer -concerns to the technology foundation to configure, share, and interpret exponentially increasing -volumes city data and services. Finally, the Infrastructure layer concerns with the base level -connectivity that supports the platform to be scalable and reliable in the long run. - - - - STACK OUTPUT - - Requirements to deliver new digital services that will address the - Societal Needs societal needs of cities in a positive manner that relates to political - narratives. - - - Services & Requirements to new profitable business models and the development - Business Models of an increase range of new and engaging services in the smart cities. - - - - City Data Requirements to provide all city data stakeholders ready access and - delivery of all city data that unpins the decision making process in smart - cities. - - - Urban Platform Requirements to put in place applications together to build a foundation - for the widespread exploitation of data. - - - Infrastructure Requirements to deliver the backbone infrastructure that will be used to - capture the opportunities of digital technology and data to enable - transformation. - - - Figure 4. Urban Platform Development Stack. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 8 - - - Table 1. Urban Platform Development Stack - Layer Rationale - Societal Needs - Accessible services and data necessary to solve social problems - and drive innovation; - - Parameters that influence user’s experience while interacting with - services (e.g. usability, feeling of security and trust); - Services and Business - Tailor-made data services which careful targets the needs of users - Models and businesses; - - New potential and cost-effective beneficial services that could be - rolled out across cities of different sizes; - - Use cases where data is used to deliver different forms of value. - City Data - Data architectural features (e.g. volume, variety, temporal factors - and sensitivity); - - Data licensing, policies and regulations to exploit data to full effect; - - Minimum metadata requirements; - - Data usability and reusability aspects of humans and machines. - Urban Platform - Holistic and interoperable solutions; - - Integrated approaches which ensures that services fit together and - that synergies can be exploited; - - Data management mechanisms to ensure data integrity and - compliance with data protection regulations - - Extension capabilities to accommodate additional functionality at - later stage at a fair and transparent cost. - - - -1.8 User Classes, Characteristics, and User Access -The users of the Urban Platform include end-users, such as the general Public, public and private -organisations; data providers; service providers; and the platform providers who will be -working with the providers of city data and services, and managing the content, defining policies -and regulations of the platform. A crucial feature of an urban platform is the provision of the -various access levels required by the different types of users. Particular uses need different -access levels to some data than the general public. Data publishers will require access to the -Urban Platform in order to ingest, administer, manage, preserve and access their resources. This -will require multiple levels of access to city data and its respective metadata. Table 2 provides a -description of each class of users. - - - Table 2. Actors - User Class Rationale - Platform - Maintains the ecosystem of data, services and users; - Provider - Defines standards, licenses and regulations and provides terms and conditions - for platform usage and the commercial exploitation of data and services; - - Decides who are allowed to join the value network of data and services - providers; - City Data - Publishes open and proprietary data into the platform; - Publisher - Manages and maintain resources in the platform accordingly to terms and - conditions. - Data - Deploys open and commercial data services into the platform (e.g. data - Services visualisation, data cleansing, data integration tools); - Provider - Manages and maintain resources in the platform accordingly to terms and - conditions. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 9 - - - City Data - Consumes open and proprietary data provided in the platform; - Consumer - Uses open and commercial data services provided in the platform; - - Provides feedback on data and services provision; - - -1.8.1 End-User Access - -City data consumers will need to access and use the city data residing in the Urban Platform. End- -users will be able to search metadata and full text within datasets (when available), and obtain city -data in open formats readily available to both humans and machines such as CSV, XML, JSON. -Some end-users may require different access rights to city data. The 2 major end-user groups that -have been identified are: - -• Open data users, including both national and international users (humans and machines). - Open access to some city data may be restricted by licensing terms (e.g. commercial data), - embargo periods, copyright, etc. - -• Private data users, which need to use the Urban Platform to obtain commercial city data. Data - access is available via data subscriptions or when purchase requirements and licenses are - waived by the data provider. - - -1.8.2 City Data Publisher Access - -A broad data provider level access is needed for stakeholders (humans and machines) working -with the urban platform and their respective data in it. Basically, data publishers will carry out the -following activities: -• Data publication access, available to publishers adding new data and metadata, checking the - quality of datasets, manipulating data, performing format conversions, defining data-access - level, tariff for consumption when applicable, and licences. - -• Data maintenance access, for publishers reviewing or editing appropriate data and metadata in - the urban platform. Data publishers can view data and add to or edit metadata without - changing the data itself. They should be provided with access to feedback from users to - investigate problem in their resources (e.g. missing data, inconsistent metadata), and statistical - information about how their resources are used by users. - - -1.8.3 Data Services Provider Access - -This is the second most restrictive access level providing rights to deploy services in the platform. -Basically, data service providers will carry out the following activities: - -• Data services deployment access, available to service providers adding new mechanisms or - integrating new applications, testing and validating integration, defining data-access level and - tariff for service usage. - -• Data services maintenance access, for services providers reviewing, extending or editing - applications in the urban platform. Data services providers can view their services deployed - and add to or edit access level and tariff without having to deploy the services again. They - should be provided with access to feedback from users to investigate problem in their services - (e.g. bugs, scalability issues), and statistical information about how their services are used by - users. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 10 - - -1.8.4 Platform Provider Access - -This is the most restrictive access level providing ultimate rights to the system and is required for -its management, development, and assigning appropriate rights to data and services providers. -Policies and regulations, license agreements are also defined by the provider of the urban -platform. Platform providers should be provided with the means to follow up on civic engagement -(e.g. feedback, request for city data) and on the provision of city data and services. - - -1.9 User Documentation -• City Data Consumers: Provide license terms and conditions associated to consuming - data and services provided in the platform, documentation of APIs and guide to - discover city data in the platform. -• City Data Providers: Provide data publication documentation describing the minimum - metadata requirements, formats accepted, step-by-step guide to publish accurate city - data. -• Data Service Providers: Disclosure technical and architecture blueprint details in - order share and outsource expertise, and partnerships, and integrate supporting - partners’ solutions into the platform itself. - - -1.10 Design and Implementation Constraints -1.10.1 Design Constraints - -• Lack of standards agreement for metadata representation. -• City data found in existing data catalogues may require special consideration concerning the - type of formats and datasets that must be stored within the platform. -• Requirements mismatch due to increased number of stakeholders involved in the design - - -1.10.2 Implementation Constraints - -• Evaluation and testing of software options is expected to occur prior to selection and - implementation of a production urban platform. -• Budget costs are unknown until evaluation of software options is completed. - - -1.11 Assumptions, Alignment with other Action Clusters and Policies -1.11.1 Assumptions - -The assumptions in Table 3 have been identified by the Demand Side Engagement Stream as -relevant to this Requirements Specification. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 11 - - -Table 3. Assumptions - - # ASSUMPTION - - 1 The providers of city data and services will be responsible to maintain their resources in the platform. - - 2 All city data must meet the minimum metadata requirements and use the standards adopted by the platform. - - 3 The platform shall consider open Source as an optional commercial model, with open standards as a principle - - 4 The system design and architecture should minimize fragmentation of city data in the urban platform. - - 5 To the extent possible, automation should be used for the extraction of descriptive and technical metadata. - - 6 The platform must be designed in a way it accommodates additional functionality at later stage at a fair and transparent cost. - - The platform must be a modular based architecture which relies on stable and well-defined open interfaces to ensure - 7 - interoperability between the platform, services and the applications provided by service providers. - - The platform will offer open and well-documented API’s and clear service descriptions and contracts that is offered for reuse by - 8 another party to foster open innovation in the city, which means that developers and interested individuals openly utilize the - resources provided. - - 9 Adopt open and published European and International standards where possible. - - The platform must be flexible enough to accommodate different local, National and International data protection, licensing and -10 - commercialization regulations. - -11 Platform providers will monitor emerging technologies in order to maintain and improve the architecture. - - -12 Platform providers will monitor emerging information standards, including metadata standards and data interface standards. - - -13 Platform providers will monitor new commercial models for city data exploitation - - - - -1.11.2 Alignment with Citizen’s Focus Action Cluster -This specification document is aligned with the principles defined in the Citizen Focus5 Action -Cluster of the European Innovation Partnership on Smart Cities and Communities. Citizen Focus’ is -about “working together with citizens to realize public interests at the intersection of ICT, mobility -and energy in an urban environment”. We recognize citizens as owners of and participants in -the creation and delivery of city data and digital services, and we specify requirements to deliver -new digital services that will address the societal needs of cities in a positive manner that relates to -political narratives. Societal needs and wants are considered the starting point for city data service -offering by the urban platform. The requirements in this document were elicited considering - -• Human behaviour and needs as important as technology; -• The services and data that solve social problems and drive innovation; -• The mechanisms that make data and services more accessible to users; -• The factors that influences user’s experience while interacting with services provided (e.g. - usability, feeling of security and trust); - - -5 - https://eu-smartcities.eu/node/1333 - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 12 - - -1.11.3 Policies to be developed - -The following policies have been identified by the Demand Side Engagement Stream as relevant to -this Requirements Specification. - -• Data formatting and Metadata Schemes: Urban platforms will require more expansive, robust - and useful data encoding and conversion that what is available in existing data catalogues. - Data preservation policies should be developed to allow data to be stored in formats that can - be migrated, associated with metadata and ontologies to become both humans and machines - readable and understandable, ongoing monitoring for data obsolescence, and migrating data to - systems environments as needed to ensure their continued availability. Current Metadata - schemes (e.g. open data, sensory data ontologies) should be reviewed to see if it meets - current needs for city data management. It is possible that the needs of the urban platform - designed here may require new or additional schemas. - -• Data commercialisation: The commercial exploitation of city data and their funding models - are unexplored concepts that we are committed to address. There is an urgent need to define - license agreements and fair commercial and subscription models to allow interoperable open - and proprietary data to co-exist in the platform. - -• Data publication and services deployment: Policy development will be needed regarding - ingesting data and deploying data services into the platform, including which users/machines - will be authorized to submit data for publication, the minimum requirements for data submitted - by open and proprietary data publishers, and the removal of resources and services from the - platform. - - -2. Urban Platform Value Proposition, Use Cases and -Functional Requirements -2.1 From Value Proposition to Platform Specifications -This document uses goal-oriented modelling for eliciting, elaborating, structuring, specifying, -documenting, and modifying requirements. Goals represent the objectives which the urban -platform should achieve through cooperation of actors in the intended system and in the -environment. They capture, at different levels of abstraction, the various objectives the urban -platform under design should achieve. Through goals modelling we consider how the value -proposition and intended solutions connects across the stack, how the urban platform meets city -goals, why the system and its functionality are needed, and how the stakeholders’ interests may be -addressed. - -In our specification, we present the overall goal (the value proposition) that the urban platform -should aim to achieve in order to be considered as a viable final product, and a set of sub-goals -(intended solutions) it should maintain in the long run so that the overall goal can be unceasingly -achieved. By using this approach, the low-level technical requirements can be traced back to high- -level strategic objectives of the urban platform. The formal notations used in this document are: -Achieve [“Name of Overall Goal”] and Maintain [“Name of Sub-Goal”]. The requirements of the -Urban Platform are noted for each of the sub-goals, and are presented as a series of statements -regarding the capabilities needed in to achieve the overall goal of the Urban Platform. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 13 - - -2.2 Overall Goal: “City data is exploited to its full potential” -An urban platform is an organization of people and systems, which has accepted the responsibility -to preserve city data and make it available for all the stakeholders of smart cities. Ultimately, an -urban platform is a foundation for the full exploitation of city data. Hence, the major goal an Urban -Platform must achieve is “city data is exploited to its full potential”. To achieve this high level goal, -the urban platform must maintain in the long run the five sub-goals illustrated in Figure 3. - -Each one of the defined sub-goals co-enables the achievement of the specified high-level (overall) -goal of the urban platform. The sub-goals include the ingestion of city data, metadata generation, -data management, data storage, access, preservation, and administration, provision of engaging -services in the smart cities. These sub-goals are discussed in details in the following sections. - - - - Achieve [City data is exploited How we achieve the platform overall goal? - Why do we need the sub-goals? - to its full effect] - Goal - - - - - Maintain [User s experience is - Maintain [City data is provided - enhanced by the provision of - in a harmonised way] - S ub-Goal 1 S ub-Goal 5 value-added services] - co-ena bles - - - - - Maintain [Resources are managed in Maintain [City data is offered - a safe and intelligent manner] in an accessible manner] - S ub-Goal 2 S ub-Goal 4 - - - - - Maintain [City data is orchestrated - in a market place] - S ub-Goal 3 - - - - - Figure 3. Platform High-Level Goal and its respective sub-goals. - - - -2.2.1 Urban Platform Boundary - -The use case diagram illustrated in Figure 4 identifies the boundaries between the actors (either -automated or human) and the urban platform. We have arrived at the urban platform boundary by -inspecting each business use case and determining, in conjunction with the stakeholders needs, -which part of the business use case should be implemented and what part should be done by an -outsourced product (e.g. Billing System) using the framework 4. - -This task is technology agnostic and takes into account the abilities of the users/actors, the -constraints, the goals of the urban platform. Table 4 maps out the use cases with their respective -sub-goals and actors. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 14 - - - - - Manage Services - - S ub-Goal 5 - - - Data Service - Publish City Data - Provider - S ub-Goal 1 - - - <> Deploy Data Services - rules - S ub-Goal 5 - - Authenticate in the - Manage Resources <> - Platform Platform - S ub-Goal 1 Provider Database - <> Manage Infrastructure System - S ub-Goal 2 - - Utlise Data Services - City Data <> - Publisher S ub-Goal 5 - - Store City Data Management - <> - S ub-Goal 2 - Systems Services - System - - <> - City Data - Consumer - QoS Monitoring - Transmit Data - System - Register in the Platform S ub-Goal 2 - - - - - <> - rules - - <> - rules - Platform - Provider - rules - - Commercialise Data - Consume City Data - Services City Data - S ub-Goal 3 S ub-Goal 4 - Consumer - <> - - - Commercialise - City Data Discover City Data - S ub-Goal 3 - S ub-Goal 4 - - Billing - Management - System - - - - Authenticate in the Register in the - Platform Platform - - - - - Authenticate Authenticate - Register Consumers Register Publishers - Consumers Publishers - - - - - Figure 4. Simplistic overview of the use cases identified in the early stages of the platform design. - - - - -Table 4. Use Cases Mapping with Sub-Goals and Actors - Sub-Goals Use Cases ID Specialised Use Cases Actors -HIGH-LEVEL GOAL: City data is exploited to its - - - - - Publish City Data UC1 User publishes city data via data API’s - City Data Publisher - 1. City data is Register as a publisher - UC2 User manually uploads datasets - provided in a in the Platform - harmonised way - full effect - - - - - User manages resources - Manage Resources UC3 City Data Publisher - User tracks resources usage - - Store City Data UC4 - 2. City data is - managed in a safe Transmit Data UC5 - Management Systems - and intelligent - manner - Manage Infrastructure UC6 - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 15 - - - City Data Publisher - Set commercial city data - Platform Provider - Subscribe to proprietary data City Data Consumer - Commercialise City - UC7 - Data - Manage commercial data City Data Publisher - - 3. City data is Manage data subscription City Data Consumer - orchestrated in a - Data Services Provider - market place Set commercial data services - Platform Provider - Subscribe to commercial services Data Services Consumer - Commercialise Data - UC8 - Services - Manage commercial services Data Services Provider - - Manage services subscription Data Services Consumer - UC9 - Register in the Platform City Data Consumer - - - 4. City data is Discover City Data City Data Consumer - UC10 - offered in an - User consumes city data via data - accessible manner - API’s - Consume City Data UC11 City Data Consumer - User downloads datasets - 5. User’s Data Services Provider - Deploy Data Services UC12 - experience is Platform Provider - enhanced by the - Manage Services UC13 - Data Services Provider - provision of value- - added services - Utilise Data Services UC14 City Data Consumer - - - - - 2.3 SUB-GOAL 1: City data is collected in an intelligent manner - Description: The urban platform enables the owners of city data to easily publish both historic and - data streams in the platform, as well as their associated metadata. - Rationale: This sub-goal is maintained by the services and functions to accept the publication of - city data from data providers (of both open and proprietary data) and prepare the contents for - storage and management within the urban platform. Functions include receiving data, performing - quality assurance on data, verifying data formatting and document standards, associating meta- - data information, and coordinating updates to databases and resources management. - - Achieve [City data is exploited - to its full effect] - Goal - co-ena bles - - - - - How we achieve the platform overall goal? - - - - Maintain [City data is provided in - a harmonised manner] - S ub-Goal 1 - - - - -What actions can maintain the sub-goal? - - - - Achieve [Register Publisher] Achieve [Publish City Data] Achieve [Manage Resources] - UC1 UC2 UC3 - - - - - Figure 5. Sub-Goal 1 “City data is collected in an intelligent manner” refinement. - - - Drivers: Ensure data is published in an easy and uniform way. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 16 - - -Actions: For Sub-Goal 1 to be maintained in the long-run it requires the efficient realisation of use -cases: “Publish City Data” and “Manage Resources”, as shown in Figure 5. - - -2.3.1 Use Case: Register Publisher -ID: UC1 -Refines: SUB-GOAL 1: City data is collected in an intelligent manner -Pre-condition: Data Publisher is not logged in the system -Actors: Data Publishers -Rationale: Data Publishers can register in the platform and request approval to submit city data. -They provide valid registration details (to be defined) and wait for registration confirmation. -Platform Providers may authorise or not data publishers to offer both open and proprietary city data -in the platform. Data submission agreement is a formal agreement between the Data Provider and -the Urban Platform defining the terms of the content, standards, metadata creation, and license -agreement. The Urban Platform will proactively work with Data Providers to agree on the content, -quality and format of city data. Agreements between Platform and Data Providers may be -renegotiated on a periodic or ad-hoc basis. -Refines into requirements: FREQ.1 to FREQ.5. - - Use Case Basic Stimulus and Responses - 1. The platform prompts the user for a username and password or - register new account. - 2. The user selects registration option. - 3. The platform prompts user for publisher registration information (e.g. - username, password, organisation) - 4. The user enters in their information. - UC1. Register 5. Platform verifies information and creates account. - Publisher o If non-valid information, platform shows error message and - returns to step 1. - 6. Platform provider is requested to approve the account - o Platform acknowledges registration has been successful - o If non approved, platform shows error message and returns to - step 1. - 7. End of registration - - -2.3.2 Use Case: Publish City Data -ID: UC2 -Refines: SUB-GOAL 1 - City data is collected in an intelligent manner -Pre-condition: User is authenticated in the platform -Actors: City data publisher -Rationale: The Publish City Data function provides the appropriate mechanisms to receive city data from -authorized data providers. Data may be manually uploaded or submitted via APIs. In general, data providers -with whom the Urban Platform negotiates submission agreements are the providers of proprietary city data -(those producing published material, i.e. publishers) and open data, and they can be either humans or -machines. The providers of the Urban Platform will provide data providers with specifications on the content, -quality and format of data, and publication terms and conditions. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 17 - - -The Publish City Data function may represent a legal transfer of custody for the data in the urban platform, -and may require that special access controls be placed on the contents. This function provides a -confirmation of receipt of data publication to the Producer, which may include a request to resubmit data in -the case of errors resulting from the submission. -Once data has arrived, it must undergo several reviews, including virus checking, format compliance, -metadata minimum requirement agreement, quality and anticipated content and data formatting. The -platform must include the ability to record all actions and decisions made concerning the publication of city -data. The reasons for publication failure (e.g. missing metadata information, non-valid dataset) will be -provided back to the city data publisher. In some cases, the provider can then resubmit corrected data and -metadata information, while in other instances data publication refusal criteria should prevent the publisher -from submitting the same dataset at a later time period (e.g. in cases of suspicious datasets – copyrights -violation, viruses). When data is successfully submitted (either via APIs or manual upload), it will be -processed/prepared for storage into the platform’s database. -Specialised Use Cases: The Use Case Publish City Data data is distinguished into two -specialised Use Cases: “User publishes city data via data API’s (UC2.1)” and “User manually -uploads datasets (UC2.2)”. -Subordinated Use Cases: “Store City Data (UC4)” -Refines into requirements: FREQ 6 to FREQ.26. - - - Specialised Use Cases Basic Stimulus and Responses - 1. Platform provides user with an interface for static data - publication - 2. User selects datasets to be uploaded - 3. User provides metadata associated with the data (license, - provenance, ownership, semantics) in accordance with defined - standards - UC2.1. User manually 4. User requests data publication - uploads datasets 5. Platform quickly process user’s request for data publication - 6. Platform validates data submitted - o If valid data, platform acknowledges data publication has - been successful - o If non-valid data, platform shows error message and returns - to step 1. - 7. End of data publication - 1. Platform provides user with an interface for real-time data - publication - 2. User input data API information - 3. User provides metadata associated with the data (license, - provenance, ownership, semantics) in accordance with defined - standards - UC2.2. User publishes 4. User confirm information and request data publication - city data via data API’s 5. Platform quickly process user’s request for data publication - 6. Platform validates data submitted - o If valid data, platform acknowledges data publication has - been successful - o If non-valid data, platform shows error message and returns - to step 1 - End of data publication - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 18 - - -2.3.3 Use Case: Manage Resources -ID: UC3 -Refines: GOAL 1: City data is collected in an intelligent manner -Pre-condition: User successfully authenticates in the platform -Actors: City data publisher -Rationale: Manage resources provides the services and functions for updating, maintaining and -accessing both data and metadata, as well as tracking the usage of resources by users. Ideally -the owners of the resources should be the only authorised user to manage resources, and other -authorised users can track the usage of the resources in the platform. The platform must provide a -database update response indicating the status of the update, avoid update errors to be -propagated in the platform, and should keep an audit trail of all actions to enable rollback. Data -usage tracking includes performing queries on the data management data to generate result sets, -and producing reports from these result sets. -Specialised Use Cases: The Use Case Manage Resources data is distinguished into two -specialised Use Cases: “User manages resources (UC3.1)” and “User tracks resources usage -(UC3.2)”. -Subordinated Use Cases: “Transmit Data (UC5)” -Refines into requirements: FREQ.27 to FREQ.25. - - - Specialised Use Cases Basic Interactions and Responses - 1. Platform provides user with an interface for resources - management (e.g. data and metadata, data usage) - 2. User chooses to edit or delete data - 3. If edit, user revise metadata associated with the data (license, - provenance, ownership, access-control, semantics); - UC3.1. User manages 4. If delete, user selects dataset(s) to be removed - resources 5. User confirms action - 6. Platform quickly process user’s request - 7. Platform confirms execution of request - o If valid request, platform acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 8. End of resources management - 1. Platform provides user with an interface for resources - management (e.g. data and metadata, data usage) - 2. User chooses to visualise usage information of a dataset - UC3.2. User tracks 3. Platform quickly process user’s request for data usage - resources usage information - 4. Platform provides user with statistical information about data - usage and data users anonymised information - 5. End of data usage tracking. - - -2.3.4 Functional Requirements - - Req. ID UC. ID Description Priority Domain - Societal Needs, - FREQ.1 UC1 Allow data publishers to register to submit data for publication Must - Platform - Tracks data publication agreements between Data and Platform Business Needs, - FREQ.2 UC1 Must - Providers Platform - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 19 - - - Store terms of agreements, and use them to monitor/review/process - FREQ.3 UC1 Must City Data, Platform - data submissions. - Able to add and edit terms of agreement, based on access of level of Business Needs, - FREQ.4 UC1 Must - user. Platform - - FREQ.5 UC1 Data publications are managed and monitored Must City Data, Platform - - Allow authenticated users from across different organisations to City Data, Platform, - FREQ.6 UC2 Must - publish city data Business Needs - Provide authorization mechanisms for users and sensors to publish - FREQ.7 UC2 Must City Data, Platform - city data - City Data, Platform, - FREQ.9 UC2 Provide mechanisms for static data publication Must - Business Needs - City Data, Platform, - FREQ.10 UC2 Provide mechanisms for real-time data publication Must - Business Needs - - FREQ.11 UC2 Enable the publication of metadata Must City Data, Platform - - - FREQ.12 UC2 Maintain temporal information about the data Must City Data, Platform - - - FREQ.13 UC2 Support sensory data collection Must City Data, Platform - - - FREQ.14 UC2 Accepts content in numerous file types/formats Must City Data, Platform - - Prompts a request for resubmission to the data provider if an error of - FREQ.15 UC2 Must City Data, Platform - data transmission or receipt occurs - - FREQ.16 UC2 Enable the semantic description of connected devices Must City Data, Platform - - - FREQ.17 UC2 Gather data from authenticated and authorized devices Must City Data, Platform - - - FREQ.18 UC2 Validates automatically the successful transfer of the data Must City Data, Platform - - - FREQ.19 UC2 Performs virus checking on data Must City Data, Platform - - Verifies the validity of the submission based on submitter, expected - FREQ.20 UC2 Must City Data, Platform - format, data quality, and completeness - Platform should have built-in checks on the incoming metadata. Data - FREQ.21 UC2 not containing the minimally defined set of attributes should be Must City Data, Platform - returned to the publisher for metadata enhancement. - System should have a user-friendly method of mapping non-standard - FREQ.22 UC2 Should City Data, Platform - metadata elements into approved standard elements. - Once ingested, metadata should be stored in a single common format. - FREQ.23 UC2 This format should be one that ensures against data loss, and allows a Must City Data, Platform - variety of access/distribution options - Data in the repository shall have sufficient technical metadata to - FREQ.24 UC2 assure functionality (e.g. viewing and display) to ensure accessibility Must City Data, Platform - and reusability. - Allows publishers to display and perform manual/visual quality control Business Needs, City - FREQ.25 UC2 Could - assurance via a user-friendly GUI Data, Platform - Business Needs, City - FREQ.26 UC2 Any errors shall prompt a request for resubmission of data Should - Data, Platform - - FREQ.27 UC3 Enable data providers to manage their resources Must Business Needs - - A minimal set of identifying information/metadata concerning data Business Needs, - FREQ.28 UC3 Must - publication submission must be recorded Platform - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 20 - - - Stores and tracks versions of data. Links /connections between - FREQ.29 UC3 Must City Data, Platform - versions are created and maintained - Give service and data providers access to anonymized data of the Business Needs - FREQ.30 UC3 Should - subscribers of their data or services - City Data, Platform, - FREQ.31 UC3 Enable data providers to maintain and repair data and metadata Should - Business Needs - Tracks data publication agreements between Data and Platform Business Needs, - FREQ.32 UC3 Must - Providers Platform - Store terms of agreements, and use them to monitor/review/process - FREQ.33 UC3 Must City Data, Platform - data submissions. - Able to add and edit terms of agreement, based on access of level of Business Needs, - FREQ.34 UC3 Must - user. Platform - - FREQ.35 UC3 Submission volumes and schedules are managed and monitored Must City Data, Platform - - - - -2.4 SUB-GOAL 2: City data is managed in a safe and intelligent manner -Rationale: The urban platform enables users to publish, consume and commercialise data, as well -as deploy and manage services all in a secure and privacy protected manner. - - Achieve [City data is exploited - to its full effect] - Goal - co-ena bles - - - - - Maintain [Resources are managed in a - safe and intelligent manner] - S ub-Goal 1 - - - - - Achieve [Store Data] Achieve [Transmit Data] Achieve [Manage Infrastructure] - UC4 UC5 UC6 - - - Figure 6. Sub-Goal 2 “City data is managed in a safe and intelligent manner” refinement. - - - -Drivers: Ensure data is secured and the identity of users are preserved -Actions: For Sub-Goal 2 to be maintained in the long-run it requires the efficient realisation of use -cases: “Store City Data” and “Retrieve and Transmit City Data”, as shown in Figure 6. - - -2.4.1 Use Case: Store City Data -ID: UC4 -Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner -Pre-condition: Data is successfully published in the platform -Actors: Urban Platform -Rationale: When data is successfully submitted (either via APIs or manual upload), it is -processed/prepared for storage into the platform’s database. This procedure will include the - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 21 - - -generation of unique identifiers to the database, enrichment with ontologies (when applicable), -encrypted (when applicable), signed with digital certificates (when applicable) to ensure that the -data conforms to the platform data formatting, standards, security and regulation. Data may be -converted to accepted formats, as needed (e.g. graph model). A primary goal of the conversion of -content for the platform is the preservation of the content. Priority will be given to preserving the -data accordingly to the policies defined in section (2.6.2). Access-control levels and license models -are associated to data which is subject to restrictions relating to access and conditions of use. -Refines into requirements: FREQ 28 – FREQ 39. - - Use Case Basic Interactions and Responses - 1. Platform mechanisms converts submitted data into a standard format - 2. Security enforcement (e.g. anonymisation, cryptography) is placed on - sensitive information. - UC4. Store City 3. Platform associates with datasets the access-control definitions set by - Data owner of resources. - 4. Data is enriched with semantics and is associated with other datasets - 5. Platform stores data in a scalable and secure database. - 6. End of data storage. - - -2.4.2 Use Case: Transmit Data -ID: UC5 -Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner -Pre-condition: Data is successfully published in the platform -Actors: Urban Platform -Rationale: The platform accepts data retrieval request, validates user’s rights to access the data, -retrieves city data from data storage, and moves a copy of the data to the relevant platform -component for further processing. If special processing is required, the retrieval function accesses -data in staging storage and applies the requested processes. The types of operations, which may -be carried out, include sub-sampling in temporal or spatial dimensions, conversions between -different data types or output formats, and other specialized processing (e.g., data visualisation). -Once it is finalised data will be sent to the appropriate delivery channels (e.g. API’s, GUI). It also -encompasses function to verify corruption during any internal data transfer. This function requires -that all hardware and software within the platform provide notification of potential errors and that -these errors are routed to standard error logs that are checked by the Platform Provider. -Refines into requirements: FREQ 40 to 54. -2.4.3 Use Case: Manage Infrastructure -ID: UC6 -Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner -Pre-condition: The platform is available -Actors: Urban Platform -Rationale: Manage infrastructure provides the services and functions for the overall operation of -the urban platform. Administration functions include monitoring quality of service agreements, -auditing data publication to ensure that they meet archive standards, and maintaining configuration -management of system hardware and software. In overall, it provides system engineering -functions to monitor and improve platform operations, and to inventory, report on, and -migrate/update the contents of the platforms’ databases. -Refines into requirements: FREQ 55 to 62. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 22 - - - - Use Case Basic Interactions and Responses - 1. Platform keeps monitoring services at run-time to ensure operation - and integrity of city data - o If system failure, the platform activates mechanisms for recovery - UC6. Manage based on pre-defined rules - Infrastructure o Platform logs issue and issue alert messages to platform providers - 2. Platform logs operation capabilities (e.g. performance, mean of time - failure, issues, etc.) - - -2.4.4 Functional Requirements - - Req. ID UC. ID Description Priority Domain - A minimal set of identifying information/metadata concerning data Business Needs, - FREQ.28 UC4 Must - publication submission must be recorded. Platform - Stores and tracks versions of data. Links /connections between - FREQ.29 UC4 Must City Data, Platform - versions are created and maintained. - - FREQ.30 UC4 Converts data to accepted file formats Must City Data, Platform - - Keep sensitive information secured and accessible only to authorized - FREQ.31 UC4 Must City Data, Platform - users - - FREQ.32 UC4 Keep user’s personal information protected Should City Data, Platform - - - FREQ.33 UC4 Keep city data and meta-data secured Must Platform Needs - - - FREQ.34 UC4 Enable privacy preserving mechanisms associated to data Must Platform Needs - - - FREQ.35 UC4 Model data in accordance with defined standards Must City Data, Platform - - - FREQ.36 UC4 Support the use of ontologies and semantic modelling of city data Could City Data - - - FREQ.37 UC4 Support database-level provenance annotation Should City Data - - - FREQ.38 UC4 Support data-level provenance annotation Should City Data - - - FREQ.39 UC4 Enable data to be encrypted Should Platform Needs - - System must have the ability to search and display metadata, preferably - City Data, Platform - FREQ.40 UC5 in a user-conformable, human readable display as well as in its native Must - Needs - format for machine harvesting and manipulation. - Controls access to data in the repository based on multiple permission - City Data, Platform - FREQ.41 UC5 levels. These permission levels determine the create/edit/read/delete Should - Needs - privileges granted users. - Access rights and conditions of use will be held for each data and its City Data, Platform - FREQ.42 UC5 Should - related metadata. Needs - Access rights and conditions can be inherited from a parent data to any City Data, Platform - FREQ.43 UC5 Could - data designated as a child data (derived information). Needs - Access rights and conditions of use will be machine readable and City Data, Platform - FREQ.44 UC5 Should - actionable. Needs - Access mechanisms must be sufficiently granular to allow the Business Needs, City - FREQ.45 UC5 identification of individual users, in order to maintain audit logs of actions Should - Data, Platform - performed by users. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 23 - - - Maintains the integrity of the database which contains both metadata - FREQ.46 UC5 and system information. Must Platform Needs - - Provides internal validation such as referential integrity of the contents of City Data, Platform - FREQ.47 UC5 the database. Must - Needs - Creates and maintains schema definitions required to support data - FREQ.48 UC5 management functions. Must Platform Needs - - Monitors and ensures that data and metadata are not corrupted during - FREQ.49 UC5 transfers. Must Platform Needs - - Provides statistically acceptable assurance that no components of the - FREQ.50 UC5 data are corrupted during any internal data transfer. Must Platform Needs - - Performs routine and special data integrity checking for each dataset - FREQ.51 UC5 and generates error reports. Must Platform Needs - - Provides disaster recovery capabilities including data backup, off-site - FREQ.52 UC5 data storage, data recovery, etc. Must Platform Needs - - Refresh/replace data without service interruption, and update - FREQ.53 UC5 corresponding metadata as appropriate. Must Platform Needs - - Ensure that any associated unique identifiers of the updated data are not - FREQ.54 UC5 altered. Must Platform Needs - - Audits submissions to ensure that they meet archive/repository - FREQ.55 UC6 standards. Must Platform Needs - - Maintains configuration management of the system hardware and - FREQ.56 UC6 software. Must Platform Needs - - Has capability to inventory, report on and migrate the contents of the - FREQ.57 UC6 repository. Must Platform Needs - - - FREQ.58 UC6 Ensures data integrity for version upgrades and format migration. Must Platform Needs - - - FREQ.59 UC6 Monitors functionality of the entire repository. Must Platform Needs - - - FREQ.60 UC6 Maintains integrity of system configuration. Must Platform Needs - - Audits system operations, performance and usage. - FREQ.61 UC6 Must Platform Needs - - Provides platform performance information and database holdings - FREQ.62 UC6 inventory reports Must Platform Needs - - - - -2.5 SUB-GOAL 3: City data is orchestrated in a marketplace -Rationale: The urban platform enables users to consume and publish data in a secure and privacy -protected manner. -Drivers: Ensure data is secured and the identity of users are preserved -Actions: For Sub-Goal 3 to be maintained in the long-run it requires the efficient realisation of use -cases: “Commercialise City Data” and “Commercialise Data Services”, as shown in Figure 7. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 24 - - - - Achieve [City data is exploited - to its full effect] - Goal - - - - - co-ena bles - Maintain [City data is orchestrated in - a marketplace] - S ub-Goal 3 - - - - - Achieve [Commercialise Data - Achieve [Commercialise City Data] - Services] - UC7 UC8 - - - - - Figure 7. Sub-Goal 3 “City data is orchestrated in a marketplace” refinement. - - - -2.5.1 Use Case: Commercialise City Data -ID: UC7 -Refines: SUB-GOAL 3: City data is orchestrated in a marketplace -Pre-condition: Data is successfully published in the platform, both publisher and consumers of -city data are authenticated in the platform, and there are billing capabilities available. -Actors: Data Publishers, Data Consumers -Rationale: The providers of city data can commercialise city data based on the policies and -financial models defined in the platform. After publishing their data, publishers can define which -data can be available as open data and which data should be available with the payment of a -subscription fee. Once publishers define which data is to be commercially exploited, the platform -will associate the data with their respective financial models and let it ready for subscription. City -data consumer chooses which data to purchase and is redirected to a billing interface where the -subscription payment is taken. The platform must provide an update response indicating the status -of the payment. If successful, data is ready available to be consumed by humans and machines, -otherwise the user can re-try the payment or cancel transaction. -Commercialise city data also involves the function of managing commercial data. It provides -services and functions for updating, maintaining and accessing both data and its respective -commercial transactions. Furthermore, it enables data providers to track the usage of commercial -data by users. Ideally the owners of the data should be the only authorised user to manage -resources, and other authorised users can track the usage of the data in the platform. Data usage -tracking includes performing queries on the data management data to generate result sets, and -producing reports from these result sets. Data consumers are provided with functions which enable -them to manage their subscriptions and financial transactions on the platform. These functions -include updating, maintaining and accessing financial transactions. For all these functions and -services, the platform must provide a database update response indicating the status of the -update, avoid update errors to be propagated in the platform, and should keep an audit trail of all -actions to enable rollback. -Specialised Use Cases: The Use Case Commercialise City Data is distinguished into four -specialised Use Cases: “Commercialise Data (UC7.1)”, “Consume Commercial City Data (UC7.2)”, -“Manage Commercial Data (UC7.3)” and “Manage Data Subscription (UC7.4)”. -Refines into requirements: FREQ 63 to 68. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 25 - - - - - Use Case Basic Interactions and Responses - 1. User selects the data to be commercialised - 2. User selects the commercial model for data consumption - UC7.1. 3. Platform validates selection - Commercialise 4. Platform associates data to subscription model - Data 5. Platform releases data for commercial exploitation in the marketplace - 6. End of data commercialisation set up. - 1. User selects the data to be subscribed to - 2. User request subscription to data - UC7.2. 3. Platform validates selection - Consume 4. Platform redirects user to billing system - Proprietary 5. Billing system deals with user request - Data o If successful, user is redirected to a GUI where data is ready to use - o If unsuccessful, user can try payment again or cancel request - 2. End of data subscription. - 1. Platform provides user with an interface for commercial data - management - 2. User chooses to edit or delete commercial data - o If edit, user revise commercial models, licenses, access-control, - semantics; - o If delete, user selects dataset(s) to be removed (following policies - UC7.3. Manage defined in the platform for data removal) - commercial 3. User confirms action - Data 4. Platform promptly process user’s request - 5. Platform confirms execution of request - o If valid request, platform acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 6. End of resources management - 1. Platform provides user with an interface for data subscription - management - 2. User chooses to edit or cancel data subscription - o If edit, user revise payment and subscription timeframe; - o If cancel, user selects dataset(s) to have subscription cancelled - UC7.4. Manage (following policies defined in the platform for data subscription) - data 3. User confirms action - subscription 4. Platform promptly process user’s request - 5. Platform confirms execution of request - o If valid request, platform updates acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 6. End of data subscription management - - - -2.5.2 Use Case: Commercialise Data Services -ID: UC8 -Refines: SUB-GOAL 3: City data is orchestrated in a marketplace -Pre-condition: Data is successfully published in the platform, both publisher and consumers of -city data are authenticated in the platform, and there are billing capabilities available. -Actors: Data Service Providers, Data Publishers and Consumers - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 26 - - -Rationale: After deploying data services in the platform, the providers of data services can choose -to commercialise services based on the policies and financial models defined in the platform. Once -data service providers define which service(s) is (are) to be commercially exploited, the platform -will associate the services with their respective financial models and let available in the platform -applications module ready for use. User (either publisher or consumer) chooses which data -services to use, and in case a charged service is selected the platform redirects the user to a -billing interface where the payment is taken. The platform must provide an update response -indicating the status of the payment. If successful, service is ready available to be used, otherwise -the user can re-try the payment or cancel transaction. Note that data service owners should be -able to waive the payment of tariff to certain users’ categories. -Data services providers are offered with functions to manage their commercial services. The -platform provides functions for updating, maintaining and accessing both service and its respective -commercial transactions. Furthermore, it enables data services providers to track the usage of -services by users. Services usage tracking includes performing queries on the platform to -generate result sets, and producing reports from these result sets. The consumers of data services -are provided with functions which enable them to manage their subscriptions and financial -transactions. These functions include updating, maintaining and accessing financial transactions. -For all these functions and services, the platform must provide a database update response -indicating the status of the update, avoid update errors to be propagated in the platform, and -should keep an audit trail of all actions to enable rollback. -Specialised Use Cases: The Use Case Commercialise Data Services is distinguished into four -specialised Use Cases: “Commercialise Data Services (UC8.1)”, “Consume Data Services -(UC8.2)”, “Manage Commercial Services (UC8.3)” and “Manage Services Subscription (UC8.4)”. -Refines into requirements: FREQ 68 to 73. - - Use Case Basic Interactions and Responses - 1. User selects the data services to be commercialised - 2. User selects the commercial model for data service usage based on the - category of users - UC8.1. 3. Platform validates selection - Commercialise 4. Platform associates data services to subscription model - Data Services 5. Platform enables data service to be commercially exploited in the - marketplace - 6. End of services commercialisation set up. - 1. User selects the data service to be subscribed to - 2. User request subscription to service - 3. Platform validates selection - UC8.2. 4. Platform validates user’s category its respective commercial models - Consume Data 4. If applicable, platform redirects user to billing system - Services 5. Billing system deals with user’s request - o If successful, user is redirected to a GUI where data is ready to use - o If unsuccessful, user can try payment again or cancel request - 6. End of data subscription. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 27 - - - 1. Platform provides user with an interface for commercial data - management - 2. User chooses to edit or delete commercial data - o If edit, user revise commercial models, licenses, access-control, - semantics; - o If delete, user selects dataset(s) to be removed (following policies - UC8.3. Manage defined in the platform for data removal) - commercial 3. User confirms action - services 4. Platform promptly process user’s request - 5. Platform confirms execution of request - o If valid request, platform acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 6. End of resources management - 1. Platform provides user with an interface for data subscription - management - 2. User chooses to edit or cancel data subscription - o If edit, user revise payment and subscription timeframe; - o If cancel, user selects dataset(s) to have subscription cancelled - UC8.4. Manage (following policies defined in the platform for data subscription) - services 3. User confirms action - subscription 4. Platform promptly process user’s request - 5. Platform confirms execution of request - o If valid request, platform updates acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 6. End of data subscription management - - -2.5.3 Functional Requirements - - Req. ID UC. ID Description Priority Domain - City Data, Platform, - FREQ.63 UC7 Support the commercialization of city data Should Business Needs - City Data, Platform, - FREQ.64 UC7 Enable users to subscribe to city data through the payment of a tariff Should Business Needs - City Data, Platform, - FREQ.65 UC7 Enable users to manage their data subscriptions Should Business - Provide platform providers mechanisms to define the terms and - FREQ.66 UC7 Must Platform Needs - conditions for platform data usage - Enable data providers to manage the subscription models of their City Data, Platform, - FREQ.67 UC7 Should - data Business - UC7/ City Data, Platform, - FREQ.68 Utilise secure and reliable billing and payment management systems Must - UC8 Business - City Data, Platform, - FREQ.69 UC8 Support the commercialization of data services Should Business Needs - Enable data providers to manage the commercial models of their City Data, Platform, - FREQ.70 UC8 Should - services Business Needs - Provide service providers mechanisms to define the terms and City Data, Platform, - FREQ.71 UC8 Should - conditions of platform services Business Needs - Allow users to pay a tariff for using certain advanced services (e.g. City Data, Platform, - FREQ.72 UC8 Should - Data manipulation, enrichment) Business Needs - City Data, Platform, - FREQ.73 UC8 Enable users to manage their data services subscriptions Should Business Needs - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 28 - - -2.6 SUB-GOAL 4: City data is offered in an accessible manner -Rationale: The urban platform provides city data in both human and machine (e.g. sensors, -actuators, systems) readable and understandable formats. -Drivers: Ensure data understandability and machine-to-machine data transaction. -Actions: For Sub-Goal 3 to be maintained in the long-run it requires the efficient realisation of use -cases: “Register Consumer”, “Discover City Data”, and “Consume City Data” as shown in Figure 8. - - Achieve [City data is exploited - to its full effect] - Goal - - - - - co-ena bles - Maintain [City data is offered in an - accessible manner] - S ub-Goal 1 - - - - - Achieve [Register Consumer] Achieve [Discover City Data] Achieve [Consume City Data] - UC9 UC10 UC11 - - - - - Figure 8. Sub-Goal 4 “City data is offered in an accessible manner” refinement. - - - -2.6.1 Use Case: Register Data Consumer -ID: UC9 -Refines: SUB-GOAL 4: City data is offered in an accessible manner -Pre-condition: User is not logged in the platform -Actors: Data Consumers -Rationale: Data Consumers can register in the platform and request approval to consume city data -via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to -confirm their registration. Users must accept the terms and conditions of platform usage and define -how their personal data can be used by the Platform Owner. Users can manage and alter their -registration information at any time they want to. -Refines into requirements: FREQ 64. - - Use Case Basic Stimulus and Responses - 1. The platform prompts the user for a username and password or register - new account. - 2. The user selects registration option. - 3. The platform prompts user for data consumer registration information - UC1. Register 4. The user enters in their information. - Consumer 5. Platform verifies information and creates account. - o If non-valid information, platform shows error message and returns to - step 1. - 6. Platform acknowledges registration has been successful - 7. End of registration - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 29 - - -2.6.2 Use Case: Discover City Data -ID: UC10 -Refines: SUB-GOAL 4: City data is offered in an accessible manner -Pre-condition: User has access to either platform GUI or API -Actors: Data Consumers -Rationale: Data Consumers can register in the platform and request approval to consume city data -via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to -confirm their registration. -Refines into requirements: FREQ 64. - - Use Case Basic Stimulus and Responses - 1. Users access specialised data query end-points (e.g. SPARQL) - 2. Users provides information for pre-defined parameters for search - 3. Users request data search - 4. Platform quickly process users request for data - UC10.1 o All queries are verified against access rights restrictions - Discover city o If restriction applies users are redirected to log in interfaces - data via data o Users provide credentials and log on the system - query end-points 5. Users are provided with query results on the end-point if access is - allowed - o If access is not not allowed platform issues an error message to the - user - 1. Users search city data via GUI - 2. Users inputs search parameters (e.g. key words, categories, formats, - publishers) - UC10.2 3. Users request data search - Discover city 4. Platform quickly process users request for data - data via data o All queries are verified against access rights restrictions - query end-points o If restriction applies users are redirected to log in interfaces - o Users provide credentials and log on the system - 5. Users are provided with query results on an interface - o If access is not allowed platform issues an error message to the - user - - - -2.6.3 Use Case: Consume City Data -ID: UC11 -Refines: SUB-GOAL 4: City data is offered in an accessible manner -Pre-condition: User has access to either platform GUI or API -Actors: Data Consumers -Rationale: Data Consumers can register in the platform and request approval to consume city data -via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to -confirm their registration. -Refines into requirements: FREQ 64. -Subordinated Use Cases: “Consume proprietary city data” - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 30 - - - - Use Case Basic Stimulus and Responses - 1. Users request data to be formatted in a particular format supported by - UC11.1 the platform - Customise City 2. Platform quickly process users request for data formatting - Data 3. Mechanism for data conversion is called and process data - 4. Users are provided with data formatted as requested - 1. Users / Machines select data to be downloaded - 2. Users / Machines are redirected to authentication mechanism in case of - UC11.2 registration is needed for the particular dataset - Consume City o If authentication is successful, users are provided with requested - Data via GUI data streams - 3. Users are provided with requested data via APIs - 1. Users / Machines makes data request on the platforms API - 2. Users / Machines are redirected to authentication mechanism in case of - UC11.3 registration is needed for the particular dataset - Consume City o If authentication is successful, users are provided with requested - Data via APIs data streams - 3. Users are provided with requested data via APIs - - - -2.6.4 Functional Requirements - - Req. ID UC. ID Description Priority Domain - Allow users to register to use services and consume proprietary city Societal Needs, - FREQ.74 UC9 Should - data and open data (optional) Platform - Keep sensitive information secured and accessible only to Societal Needs, - FREQ.75 UC9 Should - authorized users Platform - Societal Needs, - FREQ.76 UC9 Provide authentication mechanisms for users Must Platform - Societal Needs, - FREQ.77 UC9 Keep user’s personal information protected Must Platform - Allow users to control which data they are willing to provide and how Societal Needs, - FREQ.78 UC9 Must - their data should be used Platform - Societal Needs, - FREQ.79 UC10 Allow users to format data in any supported data formats Must - Platform - The query request may require data to be sourced from different Societal Needs, - FREQ.80 UC10 Must - storage locations Platform - Allows query requests against all metadata used to manage the Societal Needs, - FREQ.81 UC10 repository. Should Platform - Provide users information about the legal aspects of the data Societal Needs, - FREQ.82 UC11 Must - (license, ownership) City Data - Societal Needs, - FREQ.83 UC11 Keeps an audit trail of all actions. Must City Data - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 31 - - -2.7 SUB-GOAL 5: User’s experience is enhanced by the provision of -value-added services -Rationale: The urban platform enables users to consume and publish data in a secure and privacy -protected manner -Drivers: Ensure data is secured and the identity of users are preserved -Actions: For Sub-Goal 5 to be maintained in the long-run it requires the efficient realisation of use -cases: “Deploy Data Services”, “Manage Services”, and “Utilise Data Services” as shown in Figure -8. - - Achieve [City data is exploited - to its full effect] - Goal - - - - - co-ena bles - Maintain [User s experience is enhanced by - the provision of value-added services] - S ub-Goal 1 - - - - - Achieve [Deploy Data Services] Achieve [Manage Services] Achieve [Utilise Data Services] - UC12 UC13 UC14 - - - - - Figure 9. Sub-Goal 5 “User’s experience is enhanced by the provision of value-added services” refinement - - - -2.7.1 Deploy Data Services -ID: UC12 -Refines: SUB-GOAL 5: User’s experience is enhanced by the provision of value-added services -Pre-condition: Data Services Providers are provided with credentials, are authorised to deploy -their services in the platform, and have access to technical specifications of the platform interfaces. -Actors: Data Services Providers -Rationale: Data Services can register in the platform and request approval to deploy services in -the platform. They provide valid registration details (to be defined) and wait for registration -confirmation. Platform Providers may authorise or not data services providers to offer both open -and proprietary services in the platform. Data services providers must formally agree with service -deployment agreement with the Urban Platform. This agreement defines terms of the content, -policies, regulations, license agreement. The Urban Platform will proactively work with Service -Providers to agree on the technical specifications of interfaces and platform openness level. -Agreements between Platform and Data Service Providers may be renegotiated on a periodic or -ad-hoc basis. -Refines into requirements: FREQ.1 to FREQ.5. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 32 - - - - Use Case Basic Stimulus and Responses - 1. User authenticates in the platform - 2. User is provided access to platform interfaces for service deployment - 3. User deploy services - 4. Platform checks compatibility and any technical issues arising from the - UC12. Deploy new service - Services 5. If approved the service is ready to be used and managed in the - platform - o If deployment is not approved, platform shows error message and - returns to step 1. - 6. End of deployment - - - -2.7.2 Use Case: Manage Services -ID: UC13 -Refines: GOAL 5: User’s experience is enhanced by the provision of value-added services -Pre-condition: User successfully authenticates in the platform -Actors: Data Services Providers -Rationale: Manage services provides the functions for updating, maintaining and accessing -services as well as tracking their usage by users. Ideally the owners of the services should be the -only authorised user to manage them, and other authorised users can track the usage of the -services in the platform. In case of updates the platform must log in the database update details -and send to service providers a response indicating the status of the update. The platform should -also ensure update errors are not propagated nor affect the health of other services provided in the -platform, and should keep an audit trail of all actions to enable rollback. Data services usage -tracking includes performing queries on the data management data to generate result sets, and -producing reports from these result sets. All user’s information provided to service providers must -follow regulations of data protection and the user’s defined rules for their data use. -Specialised Use Cases: The Use Case Manage Services is distinguished into two specialised -Use Cases: “User manages services (UC13.1)” and “User tracks service usage (UC13.2)”. -Subordinated Use Cases: “Transmit Data (UC5)” -Refines into requirements: FREQ.27 to FREQ.25. - - - Specialised Use Cases Basic Interactions and Responses - 1. Platform provides user with an interface for services management - 2. User chooses to edit or delete services - 3. If edit, user revise service information (access-control, - commercial models, parameters) and deployment; - If delete, user selects services to be removed / disabled - UC13.1. User manages 4. User confirms action - services 5. Platform quickly process user’s request - 6. Platform confirms execution of request - o If valid request, platform acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 7. End of services management - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 33 - - - 1. Platform provides user with an interface for services management - 2. User chooses to visualise usage information of a service - 3. Platform quickly process user’s request for data usage - UC13.2. User tracks information - services usage 4. Platform provides user with statistical information about services - usage and data users anonymised information - 5. End of data services tracking. - - - -2.7.3 Use Case: Utilise Data Services -ID: UC2 -Refines: SUB-GOAL 1 - City data is collected in an intelligent manner -Pre-condition: User is authenticated in the platform -Actors: City data publisher -Rationale: Data Consumers can register in the platform and request approval to consume city data via GUI -or APIs. They provide valid registration details (to be defined) and wait for platform to confirm their -registration. -Subordinated Use Cases: “Commercialise Data Services (UC8)” -Refines into requirements: FREQ 6 to FREQ.26. - - - Specialised Use Cases Basic Interactions and Responses - 1. Users / Machines select data service to be utilised - 2. Users / Machines are redirected to authentication mechanism in - UC2.1. User utilises data case of registration is needed for the particular service - services o If authentication is successful, users are provided with - requested data streams - 3. Users are provided with requested service either via API or GUI - - -2.7.4 Functional Requirements - - Req. ID UC. ID Description Priority Domain - Provide stable and well-defined interfaces to ensure interoperability - Societal Needs, - FREQ.84 UC12 between the platform, services and the applications provided by Should Platform - services providers - Ensure the interfaces of the architecture are open to reduce entry Societal Needs, - FREQ.85 UC12 Should - barriers and integration issues Platform - Provide multi-purposed and network intelligent interfaces to Societal Needs, - FREQ.86 UC12 Must - providers and consumers of services Platform - Provide service providers mechanisms to define the terms and Societal Needs, - FREQ.87 UC12 Must - conditions of platform services deployment Platform - Business Needs, - FREQ.88 UC13 Provide statistical information of user’s feedback on service usage Must Platform - Allow users to use services to manipulate city data (e.g. Create Societal Needs, - FREQ.89 UC10 Must - mash ups, integrate) Platform - Allow users to provide feedback on usability, and quality of data Societal Needs, - FREQ.90 UC14 Must - and services provided by the platform Platform - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 34 - - - -3. Non-functional Requirements -3.1 Run-time Quality Requirements -3.1.1 Scalability Requirements - -Rationale: The ability of the system to execute its task within its expected performance profile and -to handle on-demand increased processing volumes of data and service requests -Drivers: Provide ready access to all data that underpins decision making processes in smart cities, -and accommodate users and data usage patterns -Refines into requirements: NFREQ.1, NFREQ.2, NFREQ.3, NFREQ.4 -Relevance: Urban platforms have ambitious performance requirements. Such platforms must cope -with user’s demand for data and services, capture real-time data that will be catalysed by a myriad -of sensors. The demand for urban platform is very likely to significantly increase over time. It is -very difficult to have clear performance characteristics due to the ubiquity, heterogeneity high -connectivity of devices and end users. - - SCALABILITY MEASURES - Actions Capture the performance requirements - Create service level agreements - Predict scalability using software simulation - Analyse the performance of the platform overtime - Conduct practical testing - Strategy Prioritize service and data requests - Distribute processing over time - Scale up or scale out as necessary - Reuse resources and results Partition and parallelize - Constantly monitor Quality of Service at runtime - - - -3.1.2 Availability and Reliability Requirements - -Rationale: The ability of the system to be fully or partly operational as and when required and to -effectively handle failures that could affect system availability -Drivers: Build a reliable foundation for “on demand” exploitation of data -Refines into requirements: NFREQ.5, NFREQ.6, NFREQ.7, NFREQ.8 -Relevance: Any system that has complex or extended availability requirements, complex recovery -processes, or a high profile (e.g., is visible to the public) - - AVAILABILITY AND RELIABILITY MEASURES - Actions Capture the availability requirements Produce the availability schedule - Estimate platform availability Estimate functional availability Assess against the - requirements Rework the architecture - Strategy Adopt fault-tolerant hardware - Use reliable infrastructure database - Log transactions - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 35 - - - Develop adaptive software to cope and recover from faults - Design and test for failure - Deploy load balancing - Identify suitable backup and disaster recovery solution - - -3.1.3 Trust Requirements - -Rationale: A quality related to the user’s belief in the reliability, integrity and ability of the functional -behaviour of the platform -Drivers: Gain understanding of what influences user’s experience while interacting with services -provided -Requirements: NFREQ.16, NFREQ.17 -Relevance: Relevant to the systems that share and collect information that may raise public -concern. In some cases, trust has to do with the reliability of data and their providers, whereas in -other cases trust can be associated with the security and privacy of the technology that was -deployed. Trust affects the reputation of the platform besides its dissemination and maturity on the -market. - -TRUST MEASURES - Capture trust requirements - Perform risk analysis so measures can be implemented -Actions Explore the vulnerability aspects of city data - Check whether extensibility requirements impact on trust - Define a trust model - Adopt trust model - Deploy monitoring capabilities and assess its impact on scalability - Manage the data in a way that ensures its compliance with data -Strategy protection regulations - Implement tampering and data misuse detection - Make use of Cryptography when necessary - Allow Federation of trust between platforms - - - -3.1.4 Security Requirements - -Rationale: Ability of the system to enforce the intended confidentiality, trust, integrity and service -and data access policies, and to detect and recover from failure in these security mechanisms. -Drivers: Manage the data and services in a way that ensures its integrity, and compliance with -data protection regulations -Relevance: Relevant to the systems that share and collect information that may raise public -concern. Urban platforms may become a valuable target for attackers which can potentially leave -huge swathes of information exposed. It could potentially undermine trust in the government and -damage its reputation. -Refines into requirements: NFREQ.9 to NFREQ.15 - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 36 - - - SECURITY MEASURES - Elicit the security requirements - Cross-check security requirements’ impact on services offered by different - providers - Verify security impact on service composition - Conduct risk analysis and impacts of security breach - Use scalable and efficient user authentication components - Use authentication and authorization components to secure interfacing with - Actions external services - Address aspects of data collection, storage and distribution - Address aspects of service and communication security among devices - Identify security requirements of the physical infrastructure - Balance and prioritize scalability (performance) and security requirements - Evaluate trade-offs between privacy considerations and preventing abuse. (While - anonymous access guarantees privacy of users, traceability of users due to abuse - of the service is not possible.) - Toughen user’s functional components - Authenticate subjects - Define and enforce access policies - Secure communication infrastructures - Secure interfaces with external systems and services - Strategy - Secure databases - Check data integrity for critical services - User digital certificates and encryption where necessary - Secure monetary transactions - Secure user’s personal information - - -3.1.5 Privacy Requirements - -Rationale: Ability of the system to ensure that the collection and transmission of personal data is -minimized and handled in accordance with user’s expectation and regulations. -Drivers: Protect the vulnerability aspects volunteered citizen’s data -Requirements: NFREQ.18, NFREQ.19, NFREQ.20, NFREQ.21, NFREQ.22 -Relevance: Ensuring users privacy is protected positively influences user’s experience, -acceptance and continuous use of the platform. Besides other factors, the reputation of the -platform depends on how well user’s information is secure and preserved. - - PRIVACY MEASURES - Capture trust requirements - Perform risk analysis so measures can be implemented - Actions Explore the vulnerability aspects of city data - Check whether extensibility requirements impact on trust - Define a trust model - Allow users to interact with the platform anonymously in given - circumstances - Manage users identification securely - Strategy Anonymize data whenever necessary to comply with data regulations - Cryptograph personal identifiers during data transmission when that is - necessary - Avoid unauthorized access to implicit information (e.g. location) - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 37 - - - Verify the impact of security, trust and scalability requirements trade-offs - on privacy - Allow the user to control how personal information is used - Empower user to control the data disclosure mechanism - - - -3.2 Non Run-time Quality Requirements -3.2.1 Evolvability Requirements - -Rationale: The ability of the platform to withstand and easily adapt when new requirements and -changes is introduced. -Drivers: Ensure the platform is able to accommodate additional functionality and emerging -technologies at later stage at a fair and transparent cost -Refines into requirements: NFREQ.23 -Relevance: Important for longer- lived and more widely used systems. Urban platforms are -expected to be highly evolvable in order to accommodate future emerging technologies and avoid -interoperability issues. - - EVOLVABILITY MEASURES - Actions Characterize and assess the evolution needs - Consider the evolution trade-offs - Balance and negotiate potential conflicting requirements emerging from - the evolution capability. - Strategy Adopt standards and open interfaces - Design loose-coupled components - Preserve the platform resilience - - - -3.2.2 Extensibility Requirements - -Rationale: The flexibility of the system to allow services and functionality to be extended and -augmented by service providers in order to increase value of services to both platform providers -and end-users. The extension of the platform services is determined by the Platform Openness -strategy. -Drivers: Stakeholders can extend the services provided by the urban platform, so that -partnerships can be built to deliver holistic and interoperable solutions. Identify integrated -approaches to design and service delivery which ensures that services fit together and that -synergies can be exploited. -Refines into requirements: NFREQ.24, NFREQ.25 -Relevance: Important for longer- lived and more widely used systems. Urban platforms are -expected to be highly evolvable in order to accommodate future emerging technologies and avoid -interoperability issues. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 38 - - - EXTENSIBILITY - Actions Characterize and assess the extensibility needs - Trace the impacts and cost of extensions - Negotiate potential conflicting requirements emerging from adding extensions - in the platform - Strategy Develop a modular architecture with standard interfaces that reduces entry - barriers due to increased transparency and integration - Share technical information about interfaces will help service providers in - targeting opportunities around the platform - - - -3.3 List of Non-Functional Requirements - -ID # Description Concern Priority Domain - - Platform, -NFREQ.1 Support different service level agreements (SLA) Scalability Should - Infrastructure - Platform, -NFREQ.2 Process services and events on a set of distributed nodes Scalability Should - Infrastructure - Platform, -NFREQ.3 Continuously monitor Quality of Service at runtime Scalability Should - Infrastructure - Platform, -NFREQ.4 Balance its load at runtime Scalability Should - Infrastructure - Platform, -NFREQ.5 Provide high availability Availability Should - Infrastructure - Platform, -NFREQ.6 Guarantee infrastructure availability Availability Should - Infrastructure - Platform, -NFREQ.7 Ensure network availability Availability Should - Infrastructure - Platform, -NFREQ.8 Be able to perform self-healing Availability Should - Infrastructure - City Data, -NFREQ.9 Expose data and services to authorized users Security Must - Platform - City Data, -NFREQ.10 Ensure services are always accessible to entitled users Security Must - Platform - -NFREQ.11 Ensure Data Freshness Security Must Platform - -NFREQ.12 Support access control mechanisms Security Must Platform - - Platform, -NFREQ.13 Have security mechanisms to protect data transmission Security Should - Infrastructure - Platform, -NFREQ.14 Make it difficult to spy on communicated messages Security Should - Infrastructure - Platform, -NFREQ.15 Be able to perform to detect threats at runtime Security Should - Infrastructure - Provide trusted and secure communication and information Platform, -NFREQ.16 Trust Should - management Infrastructure - -NFREQ.17 The platform infrastructure and services shall be trustable Trust Should Infrastructure - - Societal Needs, -NFREQ.18 Allow users to use free services anonymously Privacy Should - Platform - Societal Needs, -NFREQ.19 Allow people to use free services anonymously Privacy Should - Platform - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 39 - - - Allow users to control which data they are willing to provide Societal Needs, -NFREQ.20 Privacy Should - and how their data should be used Platform - -NFREQ.21 Keep users access-control rights/ policies secured. Privacy Should Platform - - Provide privacy protection for users interacting with the Societal Needs, -NFREQ.22 Privacy Should - platform Platform - Platform, -NFREQ.23 Provide communication confidentiality Privacy Should - Infrastructure - Societal Needs, -NFREQ.24 Be extensible for future technologies. Evolvability Must City Data, Platform, - Infrastructure - Platform, -NFREQ.26 Provide standard interfaces for service providers Extensibility Should - Business Needs - -NFREQ.26 Be able to provide services in an interoperable manner Extensibility Should Platform - - - - -4. Other Requirements -4.1 Minimal Descriptive Metadata Required from Data Provider - -Minimal Descriptive Metadata Required from Data Provider - - Attribute Required Definition Notes Examples - -Database Name Y A name given to the resource - -Author Name of a person or body associated with - Y - the creation of the resource - -Maintainer Name of the entity responsible for making - Y - the resource available - -Date Created Y Date the dataset was created - - -Date Modified Y Date the dataset was modified - - -Last Revision Y Date the dataset was last revised - - -Update Frequency Y Frequency of data maintenance - - -Mode of Release Y Open/Proprietary - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 40 - - -4.2 Urban Platform Supported Data Formats - - -MIME type Description Extensions Level - -application/json JavaScript Object Notation json supported - -application/xml eXtensible markup language file xml supported - -text/csv Comma separated values file csv supported - - - - -5. Conclusion & Forward Plans -This document represents the first set of requirements specification for urban platform. Future -activities will collaboratively assess, resolve requirements conflicts, prioritize, and validate the -requirements of the urban platform. Ultimately, this document will become a complete final -requirements specification document to guide and speed up the development open platform for -cities. Table below shows the milestones, deliverables and engagement activities completed and -yet to be completed by the Demand Side Engagement Stream. - - - - Milestones, deliverables and engagement activity Forecast date Status - Stakeholders invitation Late Completed - September/2015 - Online Workshop 1: Project Scope and Outcomes 20/11/2015 Completed - Online meeting held with city members to communicate goals, - expected outcomes, time commitment and required actions. - Start of Collaborative Requirements Engineering Process 20/11/2015 Completed - Participants reviewed the first draft of urban platform requirements, - provided comments, and suggested new requirements and changes. - Online Workshop 2: Review and Refine Requirements 08/12/2015 Completed - Participants had a conference call to review the requirements - specification document and provide their comments. - Deliverable: Requirements Specification Document v2.1 shared 05/01/2016 Completed - across the Working Streams for consultation - Online Workshop 3: Review Requirements Specification 12/01/2016 Completed - Document and discussion of Letter of Intent - Participants had a conference call to review the requirements - specification document and provide their comments, and discussed - the Letter of Intent to be signed by Mayor/Equiv of EU cities to - commit to application of common U.P. approach. - Urban Platforms Workshop in Brussels 19/01/2016 Completed - Participants provided feedback on the requirements specification - document and recommended changes in the document - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 41 - - - Deliverable: Updated Requirements Specification Document v2.2 29/01/2016 Completed - shared with Industry and Standardisation Working Streams - Deliverable: Online release of the Requirements Specification Early - Document and open calls for wider consultation on the Requirements February/2016 - - Online Workshop 4: Engagement with Industry Side Early - Capture industry feedback on the Requirements Specification February/2016 - Document to assess its suitability to drive the development of the first - draft reference architecture. - Online Workshop 5: Review Requirements Specification Mid - Document v2.2 and discuss requirement prioritization and February/2016 - balancing - Collectively determine which candidate requirements of urban - platforms should be prioritized as high importance to all the cities - (use of Value Oriented Prioritization Method). Requirements are also - prioritized to minimize risk during development of urban platforms so - that the most important requirements are made explicit and - considered in the reference architecture. - Capture of case studies from cities intended to support requirements Mid March/2016 - validation (City of Porto), and learning and confidence building - amongst cities. - \ No newline at end of file diff --git a/files/core_requirements_part1.txt b/files/core_requirements_part1.txt deleted file mode 100644 index 0e3409e..0000000 --- a/files/core_requirements_part1.txt +++ /dev/null @@ -1,250 +0,0 @@ - - - Sub-Goal 5: User’s experience is - enhanced by the provision of Use Cases: 3 Requirements: 8 - value-added services - - - - Figure 2 Structure of this document. - - - - -1.6 Product Scope and Perspective -The EIP Project’s urban platform is an open common architecture which serves for city data -collection, management and distribution. An urban platform is intended to support the widespread -exploitation of city data by humans and machines in the urban environment. Figure 3 illustrates a -holistic high level overview of the urban platform the EIP project intends to deliver. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 6 - - - -The reference architecture for urban platforms should: - -• Cater for interoperability between urban infrastructures -• Enable replicability of the solutions/platforms city to city -• Scale without technical constraints and excessive cost increase -• Provide open APIs and SDKs -• Enable Real Time capabilities -• Support implementation of functional and technical capabilities - - - - - Figure 3. High level overview of the urban platform (currently approved EC DG CNECT). - - -The current urban platform market is nascent. Many software vendors offer such a platform, though -many requirements and expectations of the stakeholders of city data are not (fully) addressed. As -a result, current urban platforms are often more costly to design and maintain, less reusable and -often not interoperable platform-to-platform, and susceptible to information fragmentation and -overload. - -The urban platform which the EIP initiative intends to design takes a step beyond the platforms -currently on the market by ensuring the requirements are fully founded on a co-created and -common set of representative city needs, from which it solicits suitable industry input, and an open -and managed collaboration between industry, cities and communities, and others, in order to take -into account their needs and concerns. To do this, it is necessary to take a technology agnostic -approach to design an open and common reference architecture for urban platforms. This platform -must ensure data is collected and sustained in accordance with well-stablished standards, -managed in a robust manner so that it can handle high level supply and demand of data, and -distributed across different value chains, systems and stakeholders. The ability to handle high level -of city data supply and demand while being user secure and accessible enough for city-wide -exploitation of data is one of many keys to the success of urban platforms. This is central to the -design and implementation of urban platforms. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 7 - - -1.7 The Urban Platform Development Stack -The requirements specification of this document is based on the Urban Platform Development -Stack illustrated in Figure 4. The stack is composed by five domains (represented as layers in the -stack) necessary to fully implement an urban platform software suite as shown in Table 1. Each -domain comprehends a set of requirements necessary to the design of a common and open urban -data service platform. The elicited requirements are used to define a technical architecture which is -simple enough to be comprehensible at least at a high level of abstraction. The platform should be -conceptually decomposable into its major subsystems, the platform’s functionality reused by many -services and external applications should be identifiable, and interactions between the platform -and services, data providers and data consumers should be well defined and explicit. - -The first layer of the stack “Societal needs” concerns to outcomes we strive for within a portfolio of -city service domains. An urban platform should recognise societal needs and wants as the starting -point for city data service offering. Ultimately, an urban platform aims to provide tailor made and -compelling engaging services for the users. The Services and Business models layers concerns -with delivering data services which carefully targets the needs and expectations of the different -users of the urban platform, and explore use cases and commercial models where data is used to -deliver different forms of value. The city data layer concerns with the mechanisms necessary to -transform urban platforms into a foundation for widespread exploitation of data, including handling -data architectural features, data usability, semantics and quality aspects. The urban platform layer -concerns to the technology foundation to configure, share, and interpret exponentially increasing -volumes city data and services. Finally, the Infrastructure layer concerns with the base level -connectivity that supports the platform to be scalable and reliable in the long run. - - - - STACK OUTPUT - - Requirements to deliver new digital services that will address the - Societal Needs societal needs of cities in a positive manner that relates to political - narratives. - - - Services & Requirements to new profitable business models and the development - Business Models of an increase range of new and engaging services in the smart cities. - - - - City Data Requirements to provide all city data stakeholders ready access and - delivery of all city data that unpins the decision making process in smart - cities. - - - Urban Platform Requirements to put in place applications together to build a foundation - for the widespread exploitation of data. - - - Infrastructure Requirements to deliver the backbone infrastructure that will be used to - capture the opportunities of digital technology and data to enable - transformation. - - - Figure 4. Urban Platform Development Stack. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 8 - - - Table 1. Urban Platform Development Stack - Layer Rationale - Societal Needs - Accessible services and data necessary to solve social problems - and drive innovation; - - Parameters that influence user’s experience while interacting with - services (e.g. usability, feeling of security and trust); - Services and Business - Tailor-made data services which careful targets the needs of users - Models and businesses; - - New potential and cost-effective beneficial services that could be - rolled out across cities of different sizes; - - Use cases where data is used to deliver different forms of value. - City Data - Data architectural features (e.g. volume, variety, temporal factors - and sensitivity); - - Data licensing, policies and regulations to exploit data to full effect; - - Minimum metadata requirements; - - Data usability and reusability aspects of humans and machines. - Urban Platform - Holistic and interoperable solutions; - - Integrated approaches which ensures that services fit together and - that synergies can be exploited; - - Data management mechanisms to ensure data integrity and - compliance with data protection regulations - - Extension capabilities to accommodate additional functionality at - later stage at a fair and transparent cost. - - - -1.8 User Classes, Characteristics, and User Access -The users of the Urban Platform include end-users, such as the general Public, public and private -organisations; data providers; service providers; and the platform providers who will be -working with the providers of city data and services, and managing the content, defining policies -and regulations of the platform. A crucial feature of an urban platform is the provision of the -various access levels required by the different types of users. Particular uses need different -access levels to some data than the general public. Data publishers will require access to the -Urban Platform in order to ingest, administer, manage, preserve and access their resources. This -will require multiple levels of access to city data and its respective metadata. Table 2 provides a -description of each class of users. - - - Table 2. Actors - User Class Rationale - Platform - Maintains the ecosystem of data, services and users; - Provider - Defines standards, licenses and regulations and provides terms and conditions - for platform usage and the commercial exploitation of data and services; - - Decides who are allowed to join the value network of data and services - providers; - City Data - Publishes open and proprietary data into the platform; - Publisher - Manages and maintain resources in the platform accordingly to terms and - conditions. - Data - Deploys open and commercial data services into the platform (e.g. data - Services visualisation, data cleansing, data integration tools); - Provider - Manages and maintain resources in the platform accordingly to terms and - conditions. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 9 - - - City Data - Consumes open and proprietary data provided in the platform; - Consumer - Uses open and commercial data services provided in the platform; - - Provides feedback on data and services provision; - - -1.8.1 End-User Access - -City data consumers will need to access and use the city data residing in the Urban Platform. End- -users will be able to search metadata and full text within datasets (when available), and obtain city -data in open formats readily available to both humans and machines such as CSV, XML, JSON. -Some end-users may require different access rights to city data. The 2 major end-user groups that -have been identified are: - -• Open data users, including both national and international users (humans and machines). - Open access to some city data may be restricted by licensing terms (e.g. commercial data), - embargo periods, copyright, etc. - -• Private data users, which need to use the Urban Platform to obtain commercial city data. Data - access is available via data subscriptions or when purchase requirements and licenses are - waived by the data provider. - - -1.8.2 City Data Publisher Access - -A broad data provider level access is needed for stakeholders (humans and machines) working -with the urban platform and their respective data in it. Basically, data publishers will carry out the -following activities: -• Data publication access, available to publishers adding new data and metadata, checking the - quality of datasets, manipulating data, performing format conversions, defining data-access - level, tariff for consumption when applicable, and licences. - -• Data maintenance access, for publishers reviewing or editing appropriate data and metadata in - the urban platform. Data publishers can view data and add to or edit metadata without - changing the data itself. They should be provided with access to feedback from users to - investigate problem in their resources (e.g. missing data, inconsistent metadata), and statistical - information about how their resources are used by users. - - -1.8.3 Data Services Provider Access - -This is the second most restrictive access level providing rights to deploy services in the platform. -Basically, data service providers will carry out the following activities: - -• Data services deployment access, available to service providers adding new mechanisms or - integrating new applications, testing and validating integration, defining data-access level and - tariff for service usage. - -• Data services maintenance access, for services providers reviewing, extending or editing - applications in the urban platform. Data services providers can view their services deployed - and add to or edit access level and tariff without having to deploy the services again. They - should be provided with access to feedback from users to investigate problem in their services - (e.g. bugs, scalability issues), and statistical information about how their services are used by - users. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 10 - - -1.8.4 Platform Provider Access - -This is the most restrictive access level providing ultimate rights to the system and is required for -its management, development, and assigning appropriate rights to data and services providers. -Policies and regulations, license agreements are also defined by the provider of the urban -platform. Platform providers should be provided with the means to follow up on civic engagement -(e.g. feedback, request for city data) and on the provision of city data and services. - - -1.9 User Documentation -• City Data Consumers: Provide license terms and conditions associated to consuming - data and services provided in the platform, documentation of APIs and guide to - discover city data in the platform. -• City Data Providers: Provide data publication documentation describing the minimum - metadata requirements, formats accepted, step-by-step guide to publish accurate city - data. -• Data Service Providers: Disclosure technical and architecture blueprint details in - order share and outsource expertise, and partnerships, and integrate supporting - partners’ solutions into the platform itself. - - -1.10 Design and Implementation Constraints -1.10.1 Design Constraints - -• Lack of standards agreement for metadata representation. diff --git a/files/core_requirements_part2.txt b/files/core_requirements_part2.txt deleted file mode 100644 index fad6368..0000000 --- a/files/core_requirements_part2.txt +++ /dev/null @@ -1,600 +0,0 @@ -• City data found in existing data catalogues may require special consideration concerning the - type of formats and datasets that must be stored within the platform. -• Requirements mismatch due to increased number of stakeholders involved in the design - - -1.10.2 Implementation Constraints - -• Evaluation and testing of software options is expected to occur prior to selection and - implementation of a production urban platform. -• Budget costs are unknown until evaluation of software options is completed. - - -1.11 Assumptions, Alignment with other Action Clusters and Policies -1.11.1 Assumptions - -The assumptions in Table 3 have been identified by the Demand Side Engagement Stream as -relevant to this Requirements Specification. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 11 - - -Table 3. Assumptions - - # ASSUMPTION - - 1 The providers of city data and services will be responsible to maintain their resources in the platform. - - 2 All city data must meet the minimum metadata requirements and use the standards adopted by the platform. - - 3 The platform shall consider open Source as an optional commercial model, with open standards as a principle - - 4 The system design and architecture should minimize fragmentation of city data in the urban platform. - - 5 To the extent possible, automation should be used for the extraction of descriptive and technical metadata. - - 6 The platform must be designed in a way it accommodates additional functionality at later stage at a fair and transparent cost. - - The platform must be a modular based architecture which relies on stable and well-defined open interfaces to ensure - 7 - interoperability between the platform, services and the applications provided by service providers. - - The platform will offer open and well-documented API’s and clear service descriptions and contracts that is offered for reuse by - 8 another party to foster open innovation in the city, which means that developers and interested individuals openly utilize the - resources provided. - - 9 Adopt open and published European and International standards where possible. - - The platform must be flexible enough to accommodate different local, National and International data protection, licensing and -10 - commercialization regulations. - -11 Platform providers will monitor emerging technologies in order to maintain and improve the architecture. - - -12 Platform providers will monitor emerging information standards, including metadata standards and data interface standards. - - -13 Platform providers will monitor new commercial models for city data exploitation - - - - -1.11.2 Alignment with Citizen’s Focus Action Cluster -This specification document is aligned with the principles defined in the Citizen Focus5 Action -Cluster of the European Innovation Partnership on Smart Cities and Communities. Citizen Focus’ is -about “working together with citizens to realize public interests at the intersection of ICT, mobility -and energy in an urban environment”. We recognize citizens as owners of and participants in -the creation and delivery of city data and digital services, and we specify requirements to deliver -new digital services that will address the societal needs of cities in a positive manner that relates to -political narratives. Societal needs and wants are considered the starting point for city data service -offering by the urban platform. The requirements in this document were elicited considering - -• Human behaviour and needs as important as technology; -• The services and data that solve social problems and drive innovation; -• The mechanisms that make data and services more accessible to users; -• The factors that influences user’s experience while interacting with services provided (e.g. - usability, feeling of security and trust); - - -5 - https://eu-smartcities.eu/node/1333 - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 12 - - -1.11.3 Policies to be developed - -The following policies have been identified by the Demand Side Engagement Stream as relevant to -this Requirements Specification. - -• Data formatting and Metadata Schemes: Urban platforms will require more expansive, robust - and useful data encoding and conversion that what is available in existing data catalogues. - Data preservation policies should be developed to allow data to be stored in formats that can - be migrated, associated with metadata and ontologies to become both humans and machines - readable and understandable, ongoing monitoring for data obsolescence, and migrating data to - systems environments as needed to ensure their continued availability. Current Metadata - schemes (e.g. open data, sensory data ontologies) should be reviewed to see if it meets - current needs for city data management. It is possible that the needs of the urban platform - designed here may require new or additional schemas. - -• Data commercialisation: The commercial exploitation of city data and their funding models - are unexplored concepts that we are committed to address. There is an urgent need to define - license agreements and fair commercial and subscription models to allow interoperable open - and proprietary data to co-exist in the platform. - -• Data publication and services deployment: Policy development will be needed regarding - ingesting data and deploying data services into the platform, including which users/machines - will be authorized to submit data for publication, the minimum requirements for data submitted - by open and proprietary data publishers, and the removal of resources and services from the - platform. - - -2. Urban Platform Value Proposition, Use Cases and -Functional Requirements -2.1 From Value Proposition to Platform Specifications -This document uses goal-oriented modelling for eliciting, elaborating, structuring, specifying, -documenting, and modifying requirements. Goals represent the objectives which the urban -platform should achieve through cooperation of actors in the intended system and in the -environment. They capture, at different levels of abstraction, the various objectives the urban -platform under design should achieve. Through goals modelling we consider how the value -proposition and intended solutions connects across the stack, how the urban platform meets city -goals, why the system and its functionality are needed, and how the stakeholders’ interests may be -addressed. - -In our specification, we present the overall goal (the value proposition) that the urban platform -should aim to achieve in order to be considered as a viable final product, and a set of sub-goals -(intended solutions) it should maintain in the long run so that the overall goal can be unceasingly -achieved. By using this approach, the low-level technical requirements can be traced back to high- -level strategic objectives of the urban platform. The formal notations used in this document are: -Achieve [“Name of Overall Goal”] and Maintain [“Name of Sub-Goal”]. The requirements of the -Urban Platform are noted for each of the sub-goals, and are presented as a series of statements -regarding the capabilities needed in to achieve the overall goal of the Urban Platform. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 13 - - -2.2 Overall Goal: “City data is exploited to its full potential” -An urban platform is an organization of people and systems, which has accepted the responsibility -to preserve city data and make it available for all the stakeholders of smart cities. Ultimately, an -urban platform is a foundation for the full exploitation of city data. Hence, the major goal an Urban -Platform must achieve is “city data is exploited to its full potential”. To achieve this high level goal, -the urban platform must maintain in the long run the five sub-goals illustrated in Figure 3. - -Each one of the defined sub-goals co-enables the achievement of the specified high-level (overall) -goal of the urban platform. The sub-goals include the ingestion of city data, metadata generation, -data management, data storage, access, preservation, and administration, provision of engaging -services in the smart cities. These sub-goals are discussed in details in the following sections. - - - - Achieve [City data is exploited How we achieve the platform overall goal? - Why do we need the sub-goals? - to its full effect] - Goal - - - - - Maintain [User s experience is - Maintain [City data is provided - enhanced by the provision of - in a harmonised way] - S ub-Goal 1 S ub-Goal 5 value-added services] - co-ena bles - - - - - Maintain [Resources are managed in Maintain [City data is offered - a safe and intelligent manner] in an accessible manner] - S ub-Goal 2 S ub-Goal 4 - - - - - Maintain [City data is orchestrated - in a market place] - S ub-Goal 3 - - - - - Figure 3. Platform High-Level Goal and its respective sub-goals. - - - -2.2.1 Urban Platform Boundary - -The use case diagram illustrated in Figure 4 identifies the boundaries between the actors (either -automated or human) and the urban platform. We have arrived at the urban platform boundary by -inspecting each business use case and determining, in conjunction with the stakeholders needs, -which part of the business use case should be implemented and what part should be done by an -outsourced product (e.g. Billing System) using the framework 4. - -This task is technology agnostic and takes into account the abilities of the users/actors, the -constraints, the goals of the urban platform. Table 4 maps out the use cases with their respective -sub-goals and actors. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 14 - - - - - Manage Services - - S ub-Goal 5 - - - Data Service - Publish City Data - Provider - S ub-Goal 1 - - - <> Deploy Data Services - rules - S ub-Goal 5 - - Authenticate in the - Manage Resources <> - Platform Platform - S ub-Goal 1 Provider Database - <> Manage Infrastructure System - S ub-Goal 2 - - Utlise Data Services - City Data <> - Publisher S ub-Goal 5 - - Store City Data Management - <> - S ub-Goal 2 - Systems Services - System - - <> - City Data - Consumer - QoS Monitoring - Transmit Data - System - Register in the Platform S ub-Goal 2 - - - - - <> - rules - - <> - rules - Platform - Provider - rules - - Commercialise Data - Consume City Data - Services City Data - S ub-Goal 3 S ub-Goal 4 - Consumer - <> - - - Commercialise - City Data Discover City Data - S ub-Goal 3 - S ub-Goal 4 - - Billing - Management - System - - - - Authenticate in the Register in the - Platform Platform - - - - - Authenticate Authenticate - Register Consumers Register Publishers - Consumers Publishers - - - - - Figure 4. Simplistic overview of the use cases identified in the early stages of the platform design. - - - - -Table 4. Use Cases Mapping with Sub-Goals and Actors - Sub-Goals Use Cases ID Specialised Use Cases Actors -HIGH-LEVEL GOAL: City data is exploited to its - - - - - Publish City Data UC1 User publishes city data via data API’s - City Data Publisher - 1. City data is Register as a publisher - UC2 User manually uploads datasets - provided in a in the Platform - harmonised way - full effect - - - - - User manages resources - Manage Resources UC3 City Data Publisher - User tracks resources usage - - Store City Data UC4 - 2. City data is - managed in a safe Transmit Data UC5 - Management Systems - and intelligent - manner - Manage Infrastructure UC6 - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 15 - - - City Data Publisher - Set commercial city data - Platform Provider - Subscribe to proprietary data City Data Consumer - Commercialise City - UC7 - Data - Manage commercial data City Data Publisher - - 3. City data is Manage data subscription City Data Consumer - orchestrated in a - Data Services Provider - market place Set commercial data services - Platform Provider - Subscribe to commercial services Data Services Consumer - Commercialise Data - UC8 - Services - Manage commercial services Data Services Provider - - Manage services subscription Data Services Consumer - UC9 - Register in the Platform City Data Consumer - - - 4. City data is Discover City Data City Data Consumer - UC10 - offered in an - User consumes city data via data - accessible manner - API’s - Consume City Data UC11 City Data Consumer - User downloads datasets - 5. User’s Data Services Provider - Deploy Data Services UC12 - experience is Platform Provider - enhanced by the - Manage Services UC13 - Data Services Provider - provision of value- - added services - Utilise Data Services UC14 City Data Consumer - - - - - 2.3 SUB-GOAL 1: City data is collected in an intelligent manner - Description: The urban platform enables the owners of city data to easily publish both historic and - data streams in the platform, as well as their associated metadata. - Rationale: This sub-goal is maintained by the services and functions to accept the publication of - city data from data providers (of both open and proprietary data) and prepare the contents for - storage and management within the urban platform. Functions include receiving data, performing - quality assurance on data, verifying data formatting and document standards, associating meta- - data information, and coordinating updates to databases and resources management. - - Achieve [City data is exploited - to its full effect] - Goal - co-ena bles - - - - - How we achieve the platform overall goal? - - - - Maintain [City data is provided in - a harmonised manner] - S ub-Goal 1 - - - - -What actions can maintain the sub-goal? - - - - Achieve [Register Publisher] Achieve [Publish City Data] Achieve [Manage Resources] - UC1 UC2 UC3 - - - - - Figure 5. Sub-Goal 1 “City data is collected in an intelligent manner” refinement. - - - Drivers: Ensure data is published in an easy and uniform way. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 16 - - -Actions: For Sub-Goal 1 to be maintained in the long-run it requires the efficient realisation of use -cases: “Publish City Data” and “Manage Resources”, as shown in Figure 5. - - -2.3.1 Use Case: Register Publisher -ID: UC1 -Refines: SUB-GOAL 1: City data is collected in an intelligent manner -Pre-condition: Data Publisher is not logged in the system -Actors: Data Publishers -Rationale: Data Publishers can register in the platform and request approval to submit city data. -They provide valid registration details (to be defined) and wait for registration confirmation. -Platform Providers may authorise or not data publishers to offer both open and proprietary city data -in the platform. Data submission agreement is a formal agreement between the Data Provider and -the Urban Platform defining the terms of the content, standards, metadata creation, and license -agreement. The Urban Platform will proactively work with Data Providers to agree on the content, -quality and format of city data. Agreements between Platform and Data Providers may be -renegotiated on a periodic or ad-hoc basis. -Refines into requirements: FREQ.1 to FREQ.5. - - Use Case Basic Stimulus and Responses - 1. The platform prompts the user for a username and password or - register new account. - 2. The user selects registration option. - 3. The platform prompts user for publisher registration information (e.g. - username, password, organisation) - 4. The user enters in their information. - UC1. Register 5. Platform verifies information and creates account. - Publisher o If non-valid information, platform shows error message and - returns to step 1. - 6. Platform provider is requested to approve the account - o Platform acknowledges registration has been successful - o If non approved, platform shows error message and returns to - step 1. - 7. End of registration - - -2.3.2 Use Case: Publish City Data -ID: UC2 -Refines: SUB-GOAL 1 - City data is collected in an intelligent manner -Pre-condition: User is authenticated in the platform -Actors: City data publisher -Rationale: The Publish City Data function provides the appropriate mechanisms to receive city data from -authorized data providers. Data may be manually uploaded or submitted via APIs. In general, data providers -with whom the Urban Platform negotiates submission agreements are the providers of proprietary city data -(those producing published material, i.e. publishers) and open data, and they can be either humans or -machines. The providers of the Urban Platform will provide data providers with specifications on the content, -quality and format of data, and publication terms and conditions. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 17 - - -The Publish City Data function may represent a legal transfer of custody for the data in the urban platform, -and may require that special access controls be placed on the contents. This function provides a -confirmation of receipt of data publication to the Producer, which may include a request to resubmit data in -the case of errors resulting from the submission. -Once data has arrived, it must undergo several reviews, including virus checking, format compliance, -metadata minimum requirement agreement, quality and anticipated content and data formatting. The -platform must include the ability to record all actions and decisions made concerning the publication of city -data. The reasons for publication failure (e.g. missing metadata information, non-valid dataset) will be -provided back to the city data publisher. In some cases, the provider can then resubmit corrected data and -metadata information, while in other instances data publication refusal criteria should prevent the publisher -from submitting the same dataset at a later time period (e.g. in cases of suspicious datasets – copyrights -violation, viruses). When data is successfully submitted (either via APIs or manual upload), it will be -processed/prepared for storage into the platform’s database. -Specialised Use Cases: The Use Case Publish City Data data is distinguished into two -specialised Use Cases: “User publishes city data via data API’s (UC2.1)” and “User manually -uploads datasets (UC2.2)”. -Subordinated Use Cases: “Store City Data (UC4)” -Refines into requirements: FREQ 6 to FREQ.26. - - - Specialised Use Cases Basic Stimulus and Responses - 1. Platform provides user with an interface for static data - publication - 2. User selects datasets to be uploaded - 3. User provides metadata associated with the data (license, - provenance, ownership, semantics) in accordance with defined - standards - UC2.1. User manually 4. User requests data publication - uploads datasets 5. Platform quickly process user’s request for data publication - 6. Platform validates data submitted - o If valid data, platform acknowledges data publication has - been successful - o If non-valid data, platform shows error message and returns - to step 1. - 7. End of data publication - 1. Platform provides user with an interface for real-time data - publication - 2. User input data API information - 3. User provides metadata associated with the data (license, - provenance, ownership, semantics) in accordance with defined - standards - UC2.2. User publishes 4. User confirm information and request data publication - city data via data API’s 5. Platform quickly process user’s request for data publication - 6. Platform validates data submitted - o If valid data, platform acknowledges data publication has - been successful - o If non-valid data, platform shows error message and returns - to step 1 - End of data publication - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 18 - - -2.3.3 Use Case: Manage Resources -ID: UC3 -Refines: GOAL 1: City data is collected in an intelligent manner -Pre-condition: User successfully authenticates in the platform -Actors: City data publisher -Rationale: Manage resources provides the services and functions for updating, maintaining and -accessing both data and metadata, as well as tracking the usage of resources by users. Ideally -the owners of the resources should be the only authorised user to manage resources, and other -authorised users can track the usage of the resources in the platform. The platform must provide a -database update response indicating the status of the update, avoid update errors to be -propagated in the platform, and should keep an audit trail of all actions to enable rollback. Data -usage tracking includes performing queries on the data management data to generate result sets, -and producing reports from these result sets. -Specialised Use Cases: The Use Case Manage Resources data is distinguished into two -specialised Use Cases: “User manages resources (UC3.1)” and “User tracks resources usage -(UC3.2)”. -Subordinated Use Cases: “Transmit Data (UC5)” -Refines into requirements: FREQ.27 to FREQ.25. - - - Specialised Use Cases Basic Interactions and Responses - 1. Platform provides user with an interface for resources - management (e.g. data and metadata, data usage) - 2. User chooses to edit or delete data - 3. If edit, user revise metadata associated with the data (license, - provenance, ownership, access-control, semantics); - UC3.1. User manages 4. If delete, user selects dataset(s) to be removed - resources 5. User confirms action - 6. Platform quickly process user’s request - 7. Platform confirms execution of request - o If valid request, platform acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 8. End of resources management - 1. Platform provides user with an interface for resources - management (e.g. data and metadata, data usage) - 2. User chooses to visualise usage information of a dataset - UC3.2. User tracks 3. Platform quickly process user’s request for data usage - resources usage information - 4. Platform provides user with statistical information about data - usage and data users anonymised information - 5. End of data usage tracking. - - -2.3.4 Functional Requirements - - Req. ID UC. ID Description Priority Domain - Societal Needs, - FREQ.1 UC1 Allow data publishers to register to submit data for publication Must - Platform - Tracks data publication agreements between Data and Platform Business Needs, - FREQ.2 UC1 Must - Providers Platform - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 19 - - - Store terms of agreements, and use them to monitor/review/process - FREQ.3 UC1 Must City Data, Platform - data submissions. - Able to add and edit terms of agreement, based on access of level of Business Needs, - FREQ.4 UC1 Must - user. Platform - - FREQ.5 UC1 Data publications are managed and monitored Must City Data, Platform - - Allow authenticated users from across different organisations to City Data, Platform, - FREQ.6 UC2 Must - publish city data Business Needs - Provide authorization mechanisms for users and sensors to publish - FREQ.7 UC2 Must City Data, Platform - city data - City Data, Platform, - FREQ.9 UC2 Provide mechanisms for static data publication Must - Business Needs - City Data, Platform, - FREQ.10 UC2 Provide mechanisms for real-time data publication Must - Business Needs - - FREQ.11 UC2 Enable the publication of metadata Must City Data, Platform - - - FREQ.12 UC2 Maintain temporal information about the data Must City Data, Platform - - - FREQ.13 UC2 Support sensory data collection Must City Data, Platform - - - FREQ.14 UC2 Accepts content in numerous file types/formats Must City Data, Platform - - Prompts a request for resubmission to the data provider if an error of diff --git a/files/core_requirements_part3.txt b/files/core_requirements_part3.txt deleted file mode 100644 index 5e39fe9..0000000 --- a/files/core_requirements_part3.txt +++ /dev/null @@ -1,600 +0,0 @@ - harmonised way - full effect - - - - - User manages resources - Manage Resources UC3 City Data Publisher - User tracks resources usage - - Store City Data UC4 - 2. City data is - managed in a safe Transmit Data UC5 - Management Systems - and intelligent - manner - Manage Infrastructure UC6 - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 15 - - - City Data Publisher - Set commercial city data - Platform Provider - Subscribe to proprietary data City Data Consumer - Commercialise City - UC7 - Data - Manage commercial data City Data Publisher - - 3. City data is Manage data subscription City Data Consumer - orchestrated in a - Data Services Provider - market place Set commercial data services - Platform Provider - Subscribe to commercial services Data Services Consumer - Commercialise Data - UC8 - Services - Manage commercial services Data Services Provider - - Manage services subscription Data Services Consumer - UC9 - Register in the Platform City Data Consumer - - - 4. City data is Discover City Data City Data Consumer - UC10 - offered in an - User consumes city data via data - accessible manner - API’s - Consume City Data UC11 City Data Consumer - User downloads datasets - 5. User’s Data Services Provider - Deploy Data Services UC12 - experience is Platform Provider - enhanced by the - Manage Services UC13 - Data Services Provider - provision of value- - added services - Utilise Data Services UC14 City Data Consumer - - - - - 2.3 SUB-GOAL 1: City data is collected in an intelligent manner - Description: The urban platform enables the owners of city data to easily publish both historic and - data streams in the platform, as well as their associated metadata. - Rationale: This sub-goal is maintained by the services and functions to accept the publication of - city data from data providers (of both open and proprietary data) and prepare the contents for - storage and management within the urban platform. Functions include receiving data, performing - quality assurance on data, verifying data formatting and document standards, associating meta- - data information, and coordinating updates to databases and resources management. - - Achieve [City data is exploited - to its full effect] - Goal - co-ena bles - - - - - How we achieve the platform overall goal? - - - - Maintain [City data is provided in - a harmonised manner] - S ub-Goal 1 - - - - -What actions can maintain the sub-goal? - - - - Achieve [Register Publisher] Achieve [Publish City Data] Achieve [Manage Resources] - UC1 UC2 UC3 - - - - - Figure 5. Sub-Goal 1 “City data is collected in an intelligent manner” refinement. - - - Drivers: Ensure data is published in an easy and uniform way. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 16 - - -Actions: For Sub-Goal 1 to be maintained in the long-run it requires the efficient realisation of use -cases: “Publish City Data” and “Manage Resources”, as shown in Figure 5. - - -2.3.1 Use Case: Register Publisher -ID: UC1 -Refines: SUB-GOAL 1: City data is collected in an intelligent manner -Pre-condition: Data Publisher is not logged in the system -Actors: Data Publishers -Rationale: Data Publishers can register in the platform and request approval to submit city data. -They provide valid registration details (to be defined) and wait for registration confirmation. -Platform Providers may authorise or not data publishers to offer both open and proprietary city data -in the platform. Data submission agreement is a formal agreement between the Data Provider and -the Urban Platform defining the terms of the content, standards, metadata creation, and license -agreement. The Urban Platform will proactively work with Data Providers to agree on the content, -quality and format of city data. Agreements between Platform and Data Providers may be -renegotiated on a periodic or ad-hoc basis. -Refines into requirements: FREQ.1 to FREQ.5. - - Use Case Basic Stimulus and Responses - 1. The platform prompts the user for a username and password or - register new account. - 2. The user selects registration option. - 3. The platform prompts user for publisher registration information (e.g. - username, password, organisation) - 4. The user enters in their information. - UC1. Register 5. Platform verifies information and creates account. - Publisher o If non-valid information, platform shows error message and - returns to step 1. - 6. Platform provider is requested to approve the account - o Platform acknowledges registration has been successful - o If non approved, platform shows error message and returns to - step 1. - 7. End of registration - - -2.3.2 Use Case: Publish City Data -ID: UC2 -Refines: SUB-GOAL 1 - City data is collected in an intelligent manner -Pre-condition: User is authenticated in the platform -Actors: City data publisher -Rationale: The Publish City Data function provides the appropriate mechanisms to receive city data from -authorized data providers. Data may be manually uploaded or submitted via APIs. In general, data providers -with whom the Urban Platform negotiates submission agreements are the providers of proprietary city data -(those producing published material, i.e. publishers) and open data, and they can be either humans or -machines. The providers of the Urban Platform will provide data providers with specifications on the content, -quality and format of data, and publication terms and conditions. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 17 - - -The Publish City Data function may represent a legal transfer of custody for the data in the urban platform, -and may require that special access controls be placed on the contents. This function provides a -confirmation of receipt of data publication to the Producer, which may include a request to resubmit data in -the case of errors resulting from the submission. -Once data has arrived, it must undergo several reviews, including virus checking, format compliance, -metadata minimum requirement agreement, quality and anticipated content and data formatting. The -platform must include the ability to record all actions and decisions made concerning the publication of city -data. The reasons for publication failure (e.g. missing metadata information, non-valid dataset) will be -provided back to the city data publisher. In some cases, the provider can then resubmit corrected data and -metadata information, while in other instances data publication refusal criteria should prevent the publisher -from submitting the same dataset at a later time period (e.g. in cases of suspicious datasets – copyrights -violation, viruses). When data is successfully submitted (either via APIs or manual upload), it will be -processed/prepared for storage into the platform’s database. -Specialised Use Cases: The Use Case Publish City Data data is distinguished into two -specialised Use Cases: “User publishes city data via data API’s (UC2.1)” and “User manually -uploads datasets (UC2.2)”. -Subordinated Use Cases: “Store City Data (UC4)” -Refines into requirements: FREQ 6 to FREQ.26. - - - Specialised Use Cases Basic Stimulus and Responses - 1. Platform provides user with an interface for static data - publication - 2. User selects datasets to be uploaded - 3. User provides metadata associated with the data (license, - provenance, ownership, semantics) in accordance with defined - standards - UC2.1. User manually 4. User requests data publication - uploads datasets 5. Platform quickly process user’s request for data publication - 6. Platform validates data submitted - o If valid data, platform acknowledges data publication has - been successful - o If non-valid data, platform shows error message and returns - to step 1. - 7. End of data publication - 1. Platform provides user with an interface for real-time data - publication - 2. User input data API information - 3. User provides metadata associated with the data (license, - provenance, ownership, semantics) in accordance with defined - standards - UC2.2. User publishes 4. User confirm information and request data publication - city data via data API’s 5. Platform quickly process user’s request for data publication - 6. Platform validates data submitted - o If valid data, platform acknowledges data publication has - been successful - o If non-valid data, platform shows error message and returns - to step 1 - End of data publication - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 18 - - -2.3.3 Use Case: Manage Resources -ID: UC3 -Refines: GOAL 1: City data is collected in an intelligent manner -Pre-condition: User successfully authenticates in the platform -Actors: City data publisher -Rationale: Manage resources provides the services and functions for updating, maintaining and -accessing both data and metadata, as well as tracking the usage of resources by users. Ideally -the owners of the resources should be the only authorised user to manage resources, and other -authorised users can track the usage of the resources in the platform. The platform must provide a -database update response indicating the status of the update, avoid update errors to be -propagated in the platform, and should keep an audit trail of all actions to enable rollback. Data -usage tracking includes performing queries on the data management data to generate result sets, -and producing reports from these result sets. -Specialised Use Cases: The Use Case Manage Resources data is distinguished into two -specialised Use Cases: “User manages resources (UC3.1)” and “User tracks resources usage -(UC3.2)”. -Subordinated Use Cases: “Transmit Data (UC5)” -Refines into requirements: FREQ.27 to FREQ.25. - - - Specialised Use Cases Basic Interactions and Responses - 1. Platform provides user with an interface for resources - management (e.g. data and metadata, data usage) - 2. User chooses to edit or delete data - 3. If edit, user revise metadata associated with the data (license, - provenance, ownership, access-control, semantics); - UC3.1. User manages 4. If delete, user selects dataset(s) to be removed - resources 5. User confirms action - 6. Platform quickly process user’s request - 7. Platform confirms execution of request - o If valid request, platform acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 8. End of resources management - 1. Platform provides user with an interface for resources - management (e.g. data and metadata, data usage) - 2. User chooses to visualise usage information of a dataset - UC3.2. User tracks 3. Platform quickly process user’s request for data usage - resources usage information - 4. Platform provides user with statistical information about data - usage and data users anonymised information - 5. End of data usage tracking. - - -2.3.4 Functional Requirements - - Req. ID UC. ID Description Priority Domain - Societal Needs, - FREQ.1 UC1 Allow data publishers to register to submit data for publication Must - Platform - Tracks data publication agreements between Data and Platform Business Needs, - FREQ.2 UC1 Must - Providers Platform - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 19 - - - Store terms of agreements, and use them to monitor/review/process - FREQ.3 UC1 Must City Data, Platform - data submissions. - Able to add and edit terms of agreement, based on access of level of Business Needs, - FREQ.4 UC1 Must - user. Platform - - FREQ.5 UC1 Data publications are managed and monitored Must City Data, Platform - - Allow authenticated users from across different organisations to City Data, Platform, - FREQ.6 UC2 Must - publish city data Business Needs - Provide authorization mechanisms for users and sensors to publish - FREQ.7 UC2 Must City Data, Platform - city data - City Data, Platform, - FREQ.9 UC2 Provide mechanisms for static data publication Must - Business Needs - City Data, Platform, - FREQ.10 UC2 Provide mechanisms for real-time data publication Must - Business Needs - - FREQ.11 UC2 Enable the publication of metadata Must City Data, Platform - - - FREQ.12 UC2 Maintain temporal information about the data Must City Data, Platform - - - FREQ.13 UC2 Support sensory data collection Must City Data, Platform - - - FREQ.14 UC2 Accepts content in numerous file types/formats Must City Data, Platform - - Prompts a request for resubmission to the data provider if an error of - FREQ.15 UC2 Must City Data, Platform - data transmission or receipt occurs - - FREQ.16 UC2 Enable the semantic description of connected devices Must City Data, Platform - - - FREQ.17 UC2 Gather data from authenticated and authorized devices Must City Data, Platform - - - FREQ.18 UC2 Validates automatically the successful transfer of the data Must City Data, Platform - - - FREQ.19 UC2 Performs virus checking on data Must City Data, Platform - - Verifies the validity of the submission based on submitter, expected - FREQ.20 UC2 Must City Data, Platform - format, data quality, and completeness - Platform should have built-in checks on the incoming metadata. Data - FREQ.21 UC2 not containing the minimally defined set of attributes should be Must City Data, Platform - returned to the publisher for metadata enhancement. - System should have a user-friendly method of mapping non-standard - FREQ.22 UC2 Should City Data, Platform - metadata elements into approved standard elements. - Once ingested, metadata should be stored in a single common format. - FREQ.23 UC2 This format should be one that ensures against data loss, and allows a Must City Data, Platform - variety of access/distribution options - Data in the repository shall have sufficient technical metadata to - FREQ.24 UC2 assure functionality (e.g. viewing and display) to ensure accessibility Must City Data, Platform - and reusability. - Allows publishers to display and perform manual/visual quality control Business Needs, City - FREQ.25 UC2 Could - assurance via a user-friendly GUI Data, Platform - Business Needs, City - FREQ.26 UC2 Any errors shall prompt a request for resubmission of data Should - Data, Platform - - FREQ.27 UC3 Enable data providers to manage their resources Must Business Needs - - A minimal set of identifying information/metadata concerning data Business Needs, - FREQ.28 UC3 Must - publication submission must be recorded Platform - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 20 - - - Stores and tracks versions of data. Links /connections between - FREQ.29 UC3 Must City Data, Platform - versions are created and maintained - Give service and data providers access to anonymized data of the Business Needs - FREQ.30 UC3 Should - subscribers of their data or services - City Data, Platform, - FREQ.31 UC3 Enable data providers to maintain and repair data and metadata Should - Business Needs - Tracks data publication agreements between Data and Platform Business Needs, - FREQ.32 UC3 Must - Providers Platform - Store terms of agreements, and use them to monitor/review/process - FREQ.33 UC3 Must City Data, Platform - data submissions. - Able to add and edit terms of agreement, based on access of level of Business Needs, - FREQ.34 UC3 Must - user. Platform - - FREQ.35 UC3 Submission volumes and schedules are managed and monitored Must City Data, Platform - - - - -2.4 SUB-GOAL 2: City data is managed in a safe and intelligent manner -Rationale: The urban platform enables users to publish, consume and commercialise data, as well -as deploy and manage services all in a secure and privacy protected manner. - - Achieve [City data is exploited - to its full effect] - Goal - co-ena bles - - - - - Maintain [Resources are managed in a - safe and intelligent manner] - S ub-Goal 1 - - - - - Achieve [Store Data] Achieve [Transmit Data] Achieve [Manage Infrastructure] - UC4 UC5 UC6 - - - Figure 6. Sub-Goal 2 “City data is managed in a safe and intelligent manner” refinement. - - - -Drivers: Ensure data is secured and the identity of users are preserved -Actions: For Sub-Goal 2 to be maintained in the long-run it requires the efficient realisation of use -cases: “Store City Data” and “Retrieve and Transmit City Data”, as shown in Figure 6. - - -2.4.1 Use Case: Store City Data -ID: UC4 -Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner -Pre-condition: Data is successfully published in the platform -Actors: Urban Platform -Rationale: When data is successfully submitted (either via APIs or manual upload), it is -processed/prepared for storage into the platform’s database. This procedure will include the - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 21 - - -generation of unique identifiers to the database, enrichment with ontologies (when applicable), -encrypted (when applicable), signed with digital certificates (when applicable) to ensure that the -data conforms to the platform data formatting, standards, security and regulation. Data may be -converted to accepted formats, as needed (e.g. graph model). A primary goal of the conversion of -content for the platform is the preservation of the content. Priority will be given to preserving the -data accordingly to the policies defined in section (2.6.2). Access-control levels and license models -are associated to data which is subject to restrictions relating to access and conditions of use. -Refines into requirements: FREQ 28 – FREQ 39. - - Use Case Basic Interactions and Responses - 1. Platform mechanisms converts submitted data into a standard format - 2. Security enforcement (e.g. anonymisation, cryptography) is placed on - sensitive information. - UC4. Store City 3. Platform associates with datasets the access-control definitions set by - Data owner of resources. - 4. Data is enriched with semantics and is associated with other datasets - 5. Platform stores data in a scalable and secure database. - 6. End of data storage. - - -2.4.2 Use Case: Transmit Data -ID: UC5 -Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner -Pre-condition: Data is successfully published in the platform -Actors: Urban Platform -Rationale: The platform accepts data retrieval request, validates user’s rights to access the data, -retrieves city data from data storage, and moves a copy of the data to the relevant platform -component for further processing. If special processing is required, the retrieval function accesses -data in staging storage and applies the requested processes. The types of operations, which may -be carried out, include sub-sampling in temporal or spatial dimensions, conversions between -different data types or output formats, and other specialized processing (e.g., data visualisation). -Once it is finalised data will be sent to the appropriate delivery channels (e.g. API’s, GUI). It also -encompasses function to verify corruption during any internal data transfer. This function requires -that all hardware and software within the platform provide notification of potential errors and that -these errors are routed to standard error logs that are checked by the Platform Provider. -Refines into requirements: FREQ 40 to 54. -2.4.3 Use Case: Manage Infrastructure -ID: UC6 -Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner -Pre-condition: The platform is available -Actors: Urban Platform -Rationale: Manage infrastructure provides the services and functions for the overall operation of -the urban platform. Administration functions include monitoring quality of service agreements, -auditing data publication to ensure that they meet archive standards, and maintaining configuration -management of system hardware and software. In overall, it provides system engineering -functions to monitor and improve platform operations, and to inventory, report on, and -migrate/update the contents of the platforms’ databases. -Refines into requirements: FREQ 55 to 62. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 22 - - - - Use Case Basic Interactions and Responses - 1. Platform keeps monitoring services at run-time to ensure operation - and integrity of city data - o If system failure, the platform activates mechanisms for recovery - UC6. Manage based on pre-defined rules - Infrastructure o Platform logs issue and issue alert messages to platform providers - 2. Platform logs operation capabilities (e.g. performance, mean of time - failure, issues, etc.) - - -2.4.4 Functional Requirements - - Req. ID UC. ID Description Priority Domain - A minimal set of identifying information/metadata concerning data Business Needs, - FREQ.28 UC4 Must - publication submission must be recorded. Platform - Stores and tracks versions of data. Links /connections between - FREQ.29 UC4 Must City Data, Platform - versions are created and maintained. - - FREQ.30 UC4 Converts data to accepted file formats Must City Data, Platform - - Keep sensitive information secured and accessible only to authorized - FREQ.31 UC4 Must City Data, Platform - users - - FREQ.32 UC4 Keep user’s personal information protected Should City Data, Platform - - - FREQ.33 UC4 Keep city data and meta-data secured Must Platform Needs - - - FREQ.34 UC4 Enable privacy preserving mechanisms associated to data Must Platform Needs - - - FREQ.35 UC4 Model data in accordance with defined standards Must City Data, Platform - - - FREQ.36 UC4 Support the use of ontologies and semantic modelling of city data Could City Data - - - FREQ.37 UC4 Support database-level provenance annotation Should City Data - - - FREQ.38 UC4 Support data-level provenance annotation Should City Data - - - FREQ.39 UC4 Enable data to be encrypted Should Platform Needs - - System must have the ability to search and display metadata, preferably - City Data, Platform - FREQ.40 UC5 in a user-conformable, human readable display as well as in its native Must - Needs - format for machine harvesting and manipulation. - Controls access to data in the repository based on multiple permission - City Data, Platform - FREQ.41 UC5 levels. These permission levels determine the create/edit/read/delete Should - Needs - privileges granted users. - Access rights and conditions of use will be held for each data and its City Data, Platform - FREQ.42 UC5 Should - related metadata. Needs - Access rights and conditions can be inherited from a parent data to any City Data, Platform - FREQ.43 UC5 Could - data designated as a child data (derived information). Needs - Access rights and conditions of use will be machine readable and City Data, Platform - FREQ.44 UC5 Should - actionable. Needs - Access mechanisms must be sufficiently granular to allow the Business Needs, City - FREQ.45 UC5 identification of individual users, in order to maintain audit logs of actions Should - Data, Platform - performed by users. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 23 - - - Maintains the integrity of the database which contains both metadata - FREQ.46 UC5 and system information. Must Platform Needs - - Provides internal validation such as referential integrity of the contents of City Data, Platform - FREQ.47 UC5 the database. Must - Needs - Creates and maintains schema definitions required to support data - FREQ.48 UC5 management functions. Must Platform Needs - - Monitors and ensures that data and metadata are not corrupted during - FREQ.49 UC5 transfers. Must Platform Needs - - Provides statistically acceptable assurance that no components of the - FREQ.50 UC5 data are corrupted during any internal data transfer. Must Platform Needs - - Performs routine and special data integrity checking for each dataset - FREQ.51 UC5 and generates error reports. Must Platform Needs - - Provides disaster recovery capabilities including data backup, off-site - FREQ.52 UC5 data storage, data recovery, etc. Must Platform Needs - - Refresh/replace data without service interruption, and update - FREQ.53 UC5 corresponding metadata as appropriate. Must Platform Needs - - Ensure that any associated unique identifiers of the updated data are not - FREQ.54 UC5 altered. Must Platform Needs - - Audits submissions to ensure that they meet archive/repository - FREQ.55 UC6 standards. Must Platform Needs - - Maintains configuration management of the system hardware and - FREQ.56 UC6 software. Must Platform Needs - - Has capability to inventory, report on and migrate the contents of the - FREQ.57 UC6 repository. Must Platform Needs - - - FREQ.58 UC6 Ensures data integrity for version upgrades and format migration. Must Platform Needs - - - FREQ.59 UC6 Monitors functionality of the entire repository. Must Platform Needs - - - FREQ.60 UC6 Maintains integrity of system configuration. Must Platform Needs - - Audits system operations, performance and usage. - FREQ.61 UC6 Must Platform Needs - - Provides platform performance information and database holdings - FREQ.62 UC6 inventory reports Must Platform Needs - - - - -2.5 SUB-GOAL 3: City data is orchestrated in a marketplace -Rationale: The urban platform enables users to consume and publish data in a secure and privacy -protected manner. -Drivers: Ensure data is secured and the identity of users are preserved -Actions: For Sub-Goal 3 to be maintained in the long-run it requires the efficient realisation of use -cases: “Commercialise City Data” and “Commercialise Data Services”, as shown in Figure 7. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 24 - - diff --git a/files/core_requirements_part4.txt b/files/core_requirements_part4.txt deleted file mode 100644 index b4cd2f1..0000000 --- a/files/core_requirements_part4.txt +++ /dev/null @@ -1,700 +0,0 @@ - city data via data API’s 5. Platform quickly process user’s request for data publication - 6. Platform validates data submitted - o If valid data, platform acknowledges data publication has - been successful - o If non-valid data, platform shows error message and returns - to step 1 - End of data publication - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 18 - - -2.3.3 Use Case: Manage Resources -ID: UC3 -Refines: GOAL 1: City data is collected in an intelligent manner -Pre-condition: User successfully authenticates in the platform -Actors: City data publisher -Rationale: Manage resources provides the services and functions for updating, maintaining and -accessing both data and metadata, as well as tracking the usage of resources by users. Ideally -the owners of the resources should be the only authorised user to manage resources, and other -authorised users can track the usage of the resources in the platform. The platform must provide a -database update response indicating the status of the update, avoid update errors to be -propagated in the platform, and should keep an audit trail of all actions to enable rollback. Data -usage tracking includes performing queries on the data management data to generate result sets, -and producing reports from these result sets. -Specialised Use Cases: The Use Case Manage Resources data is distinguished into two -specialised Use Cases: “User manages resources (UC3.1)” and “User tracks resources usage -(UC3.2)”. -Subordinated Use Cases: “Transmit Data (UC5)” -Refines into requirements: FREQ.27 to FREQ.25. - - - Specialised Use Cases Basic Interactions and Responses - 1. Platform provides user with an interface for resources - management (e.g. data and metadata, data usage) - 2. User chooses to edit or delete data - 3. If edit, user revise metadata associated with the data (license, - provenance, ownership, access-control, semantics); - UC3.1. User manages 4. If delete, user selects dataset(s) to be removed - resources 5. User confirms action - 6. Platform quickly process user’s request - 7. Platform confirms execution of request - o If valid request, platform acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 8. End of resources management - 1. Platform provides user with an interface for resources - management (e.g. data and metadata, data usage) - 2. User chooses to visualise usage information of a dataset - UC3.2. User tracks 3. Platform quickly process user’s request for data usage - resources usage information - 4. Platform provides user with statistical information about data - usage and data users anonymised information - 5. End of data usage tracking. - - -2.3.4 Functional Requirements - - Req. ID UC. ID Description Priority Domain - Societal Needs, - FREQ.1 UC1 Allow data publishers to register to submit data for publication Must - Platform - Tracks data publication agreements between Data and Platform Business Needs, - FREQ.2 UC1 Must - Providers Platform - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 19 - - - Store terms of agreements, and use them to monitor/review/process - FREQ.3 UC1 Must City Data, Platform - data submissions. - Able to add and edit terms of agreement, based on access of level of Business Needs, - FREQ.4 UC1 Must - user. Platform - - FREQ.5 UC1 Data publications are managed and monitored Must City Data, Platform - - Allow authenticated users from across different organisations to City Data, Platform, - FREQ.6 UC2 Must - publish city data Business Needs - Provide authorization mechanisms for users and sensors to publish - FREQ.7 UC2 Must City Data, Platform - city data - City Data, Platform, - FREQ.9 UC2 Provide mechanisms for static data publication Must - Business Needs - City Data, Platform, - FREQ.10 UC2 Provide mechanisms for real-time data publication Must - Business Needs - - FREQ.11 UC2 Enable the publication of metadata Must City Data, Platform - - - FREQ.12 UC2 Maintain temporal information about the data Must City Data, Platform - - - FREQ.13 UC2 Support sensory data collection Must City Data, Platform - - - FREQ.14 UC2 Accepts content in numerous file types/formats Must City Data, Platform - - Prompts a request for resubmission to the data provider if an error of - FREQ.15 UC2 Must City Data, Platform - data transmission or receipt occurs - - FREQ.16 UC2 Enable the semantic description of connected devices Must City Data, Platform - - - FREQ.17 UC2 Gather data from authenticated and authorized devices Must City Data, Platform - - - FREQ.18 UC2 Validates automatically the successful transfer of the data Must City Data, Platform - - - FREQ.19 UC2 Performs virus checking on data Must City Data, Platform - - Verifies the validity of the submission based on submitter, expected - FREQ.20 UC2 Must City Data, Platform - format, data quality, and completeness - Platform should have built-in checks on the incoming metadata. Data - FREQ.21 UC2 not containing the minimally defined set of attributes should be Must City Data, Platform - returned to the publisher for metadata enhancement. - System should have a user-friendly method of mapping non-standard - FREQ.22 UC2 Should City Data, Platform - metadata elements into approved standard elements. - Once ingested, metadata should be stored in a single common format. - FREQ.23 UC2 This format should be one that ensures against data loss, and allows a Must City Data, Platform - variety of access/distribution options - Data in the repository shall have sufficient technical metadata to - FREQ.24 UC2 assure functionality (e.g. viewing and display) to ensure accessibility Must City Data, Platform - and reusability. - Allows publishers to display and perform manual/visual quality control Business Needs, City - FREQ.25 UC2 Could - assurance via a user-friendly GUI Data, Platform - Business Needs, City - FREQ.26 UC2 Any errors shall prompt a request for resubmission of data Should - Data, Platform - - FREQ.27 UC3 Enable data providers to manage their resources Must Business Needs - - A minimal set of identifying information/metadata concerning data Business Needs, - FREQ.28 UC3 Must - publication submission must be recorded Platform - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 20 - - - Stores and tracks versions of data. Links /connections between - FREQ.29 UC3 Must City Data, Platform - versions are created and maintained - Give service and data providers access to anonymized data of the Business Needs - FREQ.30 UC3 Should - subscribers of their data or services - City Data, Platform, - FREQ.31 UC3 Enable data providers to maintain and repair data and metadata Should - Business Needs - Tracks data publication agreements between Data and Platform Business Needs, - FREQ.32 UC3 Must - Providers Platform - Store terms of agreements, and use them to monitor/review/process - FREQ.33 UC3 Must City Data, Platform - data submissions. - Able to add and edit terms of agreement, based on access of level of Business Needs, - FREQ.34 UC3 Must - user. Platform - - FREQ.35 UC3 Submission volumes and schedules are managed and monitored Must City Data, Platform - - - - -2.4 SUB-GOAL 2: City data is managed in a safe and intelligent manner -Rationale: The urban platform enables users to publish, consume and commercialise data, as well -as deploy and manage services all in a secure and privacy protected manner. - - Achieve [City data is exploited - to its full effect] - Goal - co-ena bles - - - - - Maintain [Resources are managed in a - safe and intelligent manner] - S ub-Goal 1 - - - - - Achieve [Store Data] Achieve [Transmit Data] Achieve [Manage Infrastructure] - UC4 UC5 UC6 - - - Figure 6. Sub-Goal 2 “City data is managed in a safe and intelligent manner” refinement. - - - -Drivers: Ensure data is secured and the identity of users are preserved -Actions: For Sub-Goal 2 to be maintained in the long-run it requires the efficient realisation of use -cases: “Store City Data” and “Retrieve and Transmit City Data”, as shown in Figure 6. - - -2.4.1 Use Case: Store City Data -ID: UC4 -Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner -Pre-condition: Data is successfully published in the platform -Actors: Urban Platform -Rationale: When data is successfully submitted (either via APIs or manual upload), it is -processed/prepared for storage into the platform’s database. This procedure will include the - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 21 - - -generation of unique identifiers to the database, enrichment with ontologies (when applicable), -encrypted (when applicable), signed with digital certificates (when applicable) to ensure that the -data conforms to the platform data formatting, standards, security and regulation. Data may be -converted to accepted formats, as needed (e.g. graph model). A primary goal of the conversion of -content for the platform is the preservation of the content. Priority will be given to preserving the -data accordingly to the policies defined in section (2.6.2). Access-control levels and license models -are associated to data which is subject to restrictions relating to access and conditions of use. -Refines into requirements: FREQ 28 – FREQ 39. - - Use Case Basic Interactions and Responses - 1. Platform mechanisms converts submitted data into a standard format - 2. Security enforcement (e.g. anonymisation, cryptography) is placed on - sensitive information. - UC4. Store City 3. Platform associates with datasets the access-control definitions set by - Data owner of resources. - 4. Data is enriched with semantics and is associated with other datasets - 5. Platform stores data in a scalable and secure database. - 6. End of data storage. - - -2.4.2 Use Case: Transmit Data -ID: UC5 -Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner -Pre-condition: Data is successfully published in the platform -Actors: Urban Platform -Rationale: The platform accepts data retrieval request, validates user’s rights to access the data, -retrieves city data from data storage, and moves a copy of the data to the relevant platform -component for further processing. If special processing is required, the retrieval function accesses -data in staging storage and applies the requested processes. The types of operations, which may -be carried out, include sub-sampling in temporal or spatial dimensions, conversions between -different data types or output formats, and other specialized processing (e.g., data visualisation). -Once it is finalised data will be sent to the appropriate delivery channels (e.g. API’s, GUI). It also -encompasses function to verify corruption during any internal data transfer. This function requires -that all hardware and software within the platform provide notification of potential errors and that -these errors are routed to standard error logs that are checked by the Platform Provider. -Refines into requirements: FREQ 40 to 54. -2.4.3 Use Case: Manage Infrastructure -ID: UC6 -Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner -Pre-condition: The platform is available -Actors: Urban Platform -Rationale: Manage infrastructure provides the services and functions for the overall operation of -the urban platform. Administration functions include monitoring quality of service agreements, -auditing data publication to ensure that they meet archive standards, and maintaining configuration -management of system hardware and software. In overall, it provides system engineering -functions to monitor and improve platform operations, and to inventory, report on, and -migrate/update the contents of the platforms’ databases. -Refines into requirements: FREQ 55 to 62. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 22 - - - - Use Case Basic Interactions and Responses - 1. Platform keeps monitoring services at run-time to ensure operation - and integrity of city data - o If system failure, the platform activates mechanisms for recovery - UC6. Manage based on pre-defined rules - Infrastructure o Platform logs issue and issue alert messages to platform providers - 2. Platform logs operation capabilities (e.g. performance, mean of time - failure, issues, etc.) - - -2.4.4 Functional Requirements - - Req. ID UC. ID Description Priority Domain - A minimal set of identifying information/metadata concerning data Business Needs, - FREQ.28 UC4 Must - publication submission must be recorded. Platform - Stores and tracks versions of data. Links /connections between - FREQ.29 UC4 Must City Data, Platform - versions are created and maintained. - - FREQ.30 UC4 Converts data to accepted file formats Must City Data, Platform - - Keep sensitive information secured and accessible only to authorized - FREQ.31 UC4 Must City Data, Platform - users - - FREQ.32 UC4 Keep user’s personal information protected Should City Data, Platform - - - FREQ.33 UC4 Keep city data and meta-data secured Must Platform Needs - - - FREQ.34 UC4 Enable privacy preserving mechanisms associated to data Must Platform Needs - - - FREQ.35 UC4 Model data in accordance with defined standards Must City Data, Platform - - - FREQ.36 UC4 Support the use of ontologies and semantic modelling of city data Could City Data - - - FREQ.37 UC4 Support database-level provenance annotation Should City Data - - - FREQ.38 UC4 Support data-level provenance annotation Should City Data - - - FREQ.39 UC4 Enable data to be encrypted Should Platform Needs - - System must have the ability to search and display metadata, preferably - City Data, Platform - FREQ.40 UC5 in a user-conformable, human readable display as well as in its native Must - Needs - format for machine harvesting and manipulation. - Controls access to data in the repository based on multiple permission - City Data, Platform - FREQ.41 UC5 levels. These permission levels determine the create/edit/read/delete Should - Needs - privileges granted users. - Access rights and conditions of use will be held for each data and its City Data, Platform - FREQ.42 UC5 Should - related metadata. Needs - Access rights and conditions can be inherited from a parent data to any City Data, Platform - FREQ.43 UC5 Could - data designated as a child data (derived information). Needs - Access rights and conditions of use will be machine readable and City Data, Platform - FREQ.44 UC5 Should - actionable. Needs - Access mechanisms must be sufficiently granular to allow the Business Needs, City - FREQ.45 UC5 identification of individual users, in order to maintain audit logs of actions Should - Data, Platform - performed by users. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 23 - - - Maintains the integrity of the database which contains both metadata - FREQ.46 UC5 and system information. Must Platform Needs - - Provides internal validation such as referential integrity of the contents of City Data, Platform - FREQ.47 UC5 the database. Must - Needs - Creates and maintains schema definitions required to support data - FREQ.48 UC5 management functions. Must Platform Needs - - Monitors and ensures that data and metadata are not corrupted during - FREQ.49 UC5 transfers. Must Platform Needs - - Provides statistically acceptable assurance that no components of the - FREQ.50 UC5 data are corrupted during any internal data transfer. Must Platform Needs - - Performs routine and special data integrity checking for each dataset - FREQ.51 UC5 and generates error reports. Must Platform Needs - - Provides disaster recovery capabilities including data backup, off-site - FREQ.52 UC5 data storage, data recovery, etc. Must Platform Needs - - Refresh/replace data without service interruption, and update - FREQ.53 UC5 corresponding metadata as appropriate. Must Platform Needs - - Ensure that any associated unique identifiers of the updated data are not - FREQ.54 UC5 altered. Must Platform Needs - - Audits submissions to ensure that they meet archive/repository - FREQ.55 UC6 standards. Must Platform Needs - - Maintains configuration management of the system hardware and - FREQ.56 UC6 software. Must Platform Needs - - Has capability to inventory, report on and migrate the contents of the - FREQ.57 UC6 repository. Must Platform Needs - - - FREQ.58 UC6 Ensures data integrity for version upgrades and format migration. Must Platform Needs - - - FREQ.59 UC6 Monitors functionality of the entire repository. Must Platform Needs - - - FREQ.60 UC6 Maintains integrity of system configuration. Must Platform Needs - - Audits system operations, performance and usage. - FREQ.61 UC6 Must Platform Needs - - Provides platform performance information and database holdings - FREQ.62 UC6 inventory reports Must Platform Needs - - - - -2.5 SUB-GOAL 3: City data is orchestrated in a marketplace -Rationale: The urban platform enables users to consume and publish data in a secure and privacy -protected manner. -Drivers: Ensure data is secured and the identity of users are preserved -Actions: For Sub-Goal 3 to be maintained in the long-run it requires the efficient realisation of use -cases: “Commercialise City Data” and “Commercialise Data Services”, as shown in Figure 7. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 24 - - - - Achieve [City data is exploited - to its full effect] - Goal - - - - - co-ena bles - Maintain [City data is orchestrated in - a marketplace] - S ub-Goal 3 - - - - - Achieve [Commercialise Data - Achieve [Commercialise City Data] - Services] - UC7 UC8 - - - - - Figure 7. Sub-Goal 3 “City data is orchestrated in a marketplace” refinement. - - - -2.5.1 Use Case: Commercialise City Data -ID: UC7 -Refines: SUB-GOAL 3: City data is orchestrated in a marketplace -Pre-condition: Data is successfully published in the platform, both publisher and consumers of -city data are authenticated in the platform, and there are billing capabilities available. -Actors: Data Publishers, Data Consumers -Rationale: The providers of city data can commercialise city data based on the policies and -financial models defined in the platform. After publishing their data, publishers can define which -data can be available as open data and which data should be available with the payment of a -subscription fee. Once publishers define which data is to be commercially exploited, the platform -will associate the data with their respective financial models and let it ready for subscription. City -data consumer chooses which data to purchase and is redirected to a billing interface where the -subscription payment is taken. The platform must provide an update response indicating the status -of the payment. If successful, data is ready available to be consumed by humans and machines, -otherwise the user can re-try the payment or cancel transaction. -Commercialise city data also involves the function of managing commercial data. It provides -services and functions for updating, maintaining and accessing both data and its respective -commercial transactions. Furthermore, it enables data providers to track the usage of commercial -data by users. Ideally the owners of the data should be the only authorised user to manage -resources, and other authorised users can track the usage of the data in the platform. Data usage -tracking includes performing queries on the data management data to generate result sets, and -producing reports from these result sets. Data consumers are provided with functions which enable -them to manage their subscriptions and financial transactions on the platform. These functions -include updating, maintaining and accessing financial transactions. For all these functions and -services, the platform must provide a database update response indicating the status of the -update, avoid update errors to be propagated in the platform, and should keep an audit trail of all -actions to enable rollback. -Specialised Use Cases: The Use Case Commercialise City Data is distinguished into four -specialised Use Cases: “Commercialise Data (UC7.1)”, “Consume Commercial City Data (UC7.2)”, -“Manage Commercial Data (UC7.3)” and “Manage Data Subscription (UC7.4)”. -Refines into requirements: FREQ 63 to 68. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 25 - - - - - Use Case Basic Interactions and Responses - 1. User selects the data to be commercialised - 2. User selects the commercial model for data consumption - UC7.1. 3. Platform validates selection - Commercialise 4. Platform associates data to subscription model - Data 5. Platform releases data for commercial exploitation in the marketplace - 6. End of data commercialisation set up. - 1. User selects the data to be subscribed to - 2. User request subscription to data - UC7.2. 3. Platform validates selection - Consume 4. Platform redirects user to billing system - Proprietary 5. Billing system deals with user request - Data o If successful, user is redirected to a GUI where data is ready to use - o If unsuccessful, user can try payment again or cancel request - 2. End of data subscription. - 1. Platform provides user with an interface for commercial data - management - 2. User chooses to edit or delete commercial data - o If edit, user revise commercial models, licenses, access-control, - semantics; - o If delete, user selects dataset(s) to be removed (following policies - UC7.3. Manage defined in the platform for data removal) - commercial 3. User confirms action - Data 4. Platform promptly process user’s request - 5. Platform confirms execution of request - o If valid request, platform acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 6. End of resources management - 1. Platform provides user with an interface for data subscription - management - 2. User chooses to edit or cancel data subscription - o If edit, user revise payment and subscription timeframe; - o If cancel, user selects dataset(s) to have subscription cancelled - UC7.4. Manage (following policies defined in the platform for data subscription) - data 3. User confirms action - subscription 4. Platform promptly process user’s request - 5. Platform confirms execution of request - o If valid request, platform updates acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 6. End of data subscription management - - - -2.5.2 Use Case: Commercialise Data Services -ID: UC8 -Refines: SUB-GOAL 3: City data is orchestrated in a marketplace -Pre-condition: Data is successfully published in the platform, both publisher and consumers of -city data are authenticated in the platform, and there are billing capabilities available. -Actors: Data Service Providers, Data Publishers and Consumers - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 26 - - -Rationale: After deploying data services in the platform, the providers of data services can choose -to commercialise services based on the policies and financial models defined in the platform. Once -data service providers define which service(s) is (are) to be commercially exploited, the platform -will associate the services with their respective financial models and let available in the platform -applications module ready for use. User (either publisher or consumer) chooses which data -services to use, and in case a charged service is selected the platform redirects the user to a -billing interface where the payment is taken. The platform must provide an update response -indicating the status of the payment. If successful, service is ready available to be used, otherwise -the user can re-try the payment or cancel transaction. Note that data service owners should be -able to waive the payment of tariff to certain users’ categories. -Data services providers are offered with functions to manage their commercial services. The -platform provides functions for updating, maintaining and accessing both service and its respective -commercial transactions. Furthermore, it enables data services providers to track the usage of -services by users. Services usage tracking includes performing queries on the platform to -generate result sets, and producing reports from these result sets. The consumers of data services -are provided with functions which enable them to manage their subscriptions and financial -transactions. These functions include updating, maintaining and accessing financial transactions. -For all these functions and services, the platform must provide a database update response -indicating the status of the update, avoid update errors to be propagated in the platform, and -should keep an audit trail of all actions to enable rollback. -Specialised Use Cases: The Use Case Commercialise Data Services is distinguished into four -specialised Use Cases: “Commercialise Data Services (UC8.1)”, “Consume Data Services -(UC8.2)”, “Manage Commercial Services (UC8.3)” and “Manage Services Subscription (UC8.4)”. -Refines into requirements: FREQ 68 to 73. - - Use Case Basic Interactions and Responses - 1. User selects the data services to be commercialised - 2. User selects the commercial model for data service usage based on the - category of users - UC8.1. 3. Platform validates selection - Commercialise 4. Platform associates data services to subscription model - Data Services 5. Platform enables data service to be commercially exploited in the - marketplace - 6. End of services commercialisation set up. - 1. User selects the data service to be subscribed to - 2. User request subscription to service - 3. Platform validates selection - UC8.2. 4. Platform validates user’s category its respective commercial models - Consume Data 4. If applicable, platform redirects user to billing system - Services 5. Billing system deals with user’s request - o If successful, user is redirected to a GUI where data is ready to use - o If unsuccessful, user can try payment again or cancel request - 6. End of data subscription. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 27 - - - 1. Platform provides user with an interface for commercial data - management - 2. User chooses to edit or delete commercial data - o If edit, user revise commercial models, licenses, access-control, - semantics; - o If delete, user selects dataset(s) to be removed (following policies - UC8.3. Manage defined in the platform for data removal) - commercial 3. User confirms action - services 4. Platform promptly process user’s request - 5. Platform confirms execution of request - o If valid request, platform acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 6. End of resources management - 1. Platform provides user with an interface for data subscription - management - 2. User chooses to edit or cancel data subscription - o If edit, user revise payment and subscription timeframe; - o If cancel, user selects dataset(s) to have subscription cancelled - UC8.4. Manage (following policies defined in the platform for data subscription) - services 3. User confirms action - subscription 4. Platform promptly process user’s request - 5. Platform confirms execution of request - o If valid request, platform updates acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 6. End of data subscription management - - -2.5.3 Functional Requirements - - Req. ID UC. ID Description Priority Domain - City Data, Platform, - FREQ.63 UC7 Support the commercialization of city data Should Business Needs - City Data, Platform, - FREQ.64 UC7 Enable users to subscribe to city data through the payment of a tariff Should Business Needs - City Data, Platform, - FREQ.65 UC7 Enable users to manage their data subscriptions Should Business - Provide platform providers mechanisms to define the terms and - FREQ.66 UC7 Must Platform Needs - conditions for platform data usage - Enable data providers to manage the subscription models of their City Data, Platform, - FREQ.67 UC7 Should - data Business - UC7/ City Data, Platform, - FREQ.68 Utilise secure and reliable billing and payment management systems Must - UC8 Business - City Data, Platform, - FREQ.69 UC8 Support the commercialization of data services Should Business Needs - Enable data providers to manage the commercial models of their City Data, Platform, - FREQ.70 UC8 Should - services Business Needs - Provide service providers mechanisms to define the terms and City Data, Platform, - FREQ.71 UC8 Should - conditions of platform services Business Needs - Allow users to pay a tariff for using certain advanced services (e.g. City Data, Platform, - FREQ.72 UC8 Should - Data manipulation, enrichment) Business Needs - City Data, Platform, - FREQ.73 UC8 Enable users to manage their data services subscriptions Should Business Needs - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 28 - - -2.6 SUB-GOAL 4: City data is offered in an accessible manner -Rationale: The urban platform provides city data in both human and machine (e.g. sensors, -actuators, systems) readable and understandable formats. -Drivers: Ensure data understandability and machine-to-machine data transaction. -Actions: For Sub-Goal 3 to be maintained in the long-run it requires the efficient realisation of use -cases: “Register Consumer”, “Discover City Data”, and “Consume City Data” as shown in Figure 8. - - Achieve [City data is exploited - to its full effect] - Goal - - - - - co-ena bles - Maintain [City data is offered in an - accessible manner] - S ub-Goal 1 - - - - - Achieve [Register Consumer] Achieve [Discover City Data] Achieve [Consume City Data] - UC9 UC10 UC11 - - - - - Figure 8. Sub-Goal 4 “City data is offered in an accessible manner” refinement. - - - -2.6.1 Use Case: Register Data Consumer -ID: UC9 -Refines: SUB-GOAL 4: City data is offered in an accessible manner -Pre-condition: User is not logged in the platform -Actors: Data Consumers -Rationale: Data Consumers can register in the platform and request approval to consume city data -via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to -confirm their registration. Users must accept the terms and conditions of platform usage and define -how their personal data can be used by the Platform Owner. Users can manage and alter their -registration information at any time they want to. -Refines into requirements: FREQ 64. - - Use Case Basic Stimulus and Responses - 1. The platform prompts the user for a username and password or register - new account. - 2. The user selects registration option. - 3. The platform prompts user for data consumer registration information - UC1. Register 4. The user enters in their information. - Consumer 5. Platform verifies information and creates account. - o If non-valid information, platform shows error message and returns to - step 1. - 6. Platform acknowledges registration has been successful - 7. End of registration - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 29 - - -2.6.2 Use Case: Discover City Data -ID: UC10 -Refines: SUB-GOAL 4: City data is offered in an accessible manner -Pre-condition: User has access to either platform GUI or API -Actors: Data Consumers -Rationale: Data Consumers can register in the platform and request approval to consume city data -via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to -confirm their registration. -Refines into requirements: FREQ 64. - - Use Case Basic Stimulus and Responses - 1. Users access specialised data query end-points (e.g. SPARQL) - 2. Users provides information for pre-defined parameters for search - 3. Users request data search - 4. Platform quickly process users request for data diff --git a/files/core_requirements_part5.txt b/files/core_requirements_part5.txt deleted file mode 100644 index 69b68e6..0000000 --- a/files/core_requirements_part5.txt +++ /dev/null @@ -1,604 +0,0 @@ - Consume City o If authentication is successful, users are provided with requested - Data via APIs data streams - 3. Users are provided with requested data via APIs - - - -2.6.4 Functional Requirements - - Req. ID UC. ID Description Priority Domain - Allow users to register to use services and consume proprietary city Societal Needs, - FREQ.74 UC9 Should - data and open data (optional) Platform - Keep sensitive information secured and accessible only to Societal Needs, - FREQ.75 UC9 Should - authorized users Platform - Societal Needs, - FREQ.76 UC9 Provide authentication mechanisms for users Must Platform - Societal Needs, - FREQ.77 UC9 Keep user’s personal information protected Must Platform - Allow users to control which data they are willing to provide and how Societal Needs, - FREQ.78 UC9 Must - their data should be used Platform - Societal Needs, - FREQ.79 UC10 Allow users to format data in any supported data formats Must - Platform - The query request may require data to be sourced from different Societal Needs, - FREQ.80 UC10 Must - storage locations Platform - Allows query requests against all metadata used to manage the Societal Needs, - FREQ.81 UC10 repository. Should Platform - Provide users information about the legal aspects of the data Societal Needs, - FREQ.82 UC11 Must - (license, ownership) City Data - Societal Needs, - FREQ.83 UC11 Keeps an audit trail of all actions. Must City Data - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 31 - - -2.7 SUB-GOAL 5: User’s experience is enhanced by the provision of -value-added services -Rationale: The urban platform enables users to consume and publish data in a secure and privacy -protected manner -Drivers: Ensure data is secured and the identity of users are preserved -Actions: For Sub-Goal 5 to be maintained in the long-run it requires the efficient realisation of use -cases: “Deploy Data Services”, “Manage Services”, and “Utilise Data Services” as shown in Figure -8. - - Achieve [City data is exploited - to its full effect] - Goal - - - - - co-ena bles - Maintain [User s experience is enhanced by - the provision of value-added services] - S ub-Goal 1 - - - - - Achieve [Deploy Data Services] Achieve [Manage Services] Achieve [Utilise Data Services] - UC12 UC13 UC14 - - - - - Figure 9. Sub-Goal 5 “User’s experience is enhanced by the provision of value-added services” refinement - - - -2.7.1 Deploy Data Services -ID: UC12 -Refines: SUB-GOAL 5: User’s experience is enhanced by the provision of value-added services -Pre-condition: Data Services Providers are provided with credentials, are authorised to deploy -their services in the platform, and have access to technical specifications of the platform interfaces. -Actors: Data Services Providers -Rationale: Data Services can register in the platform and request approval to deploy services in -the platform. They provide valid registration details (to be defined) and wait for registration -confirmation. Platform Providers may authorise or not data services providers to offer both open -and proprietary services in the platform. Data services providers must formally agree with service -deployment agreement with the Urban Platform. This agreement defines terms of the content, -policies, regulations, license agreement. The Urban Platform will proactively work with Service -Providers to agree on the technical specifications of interfaces and platform openness level. -Agreements between Platform and Data Service Providers may be renegotiated on a periodic or -ad-hoc basis. -Refines into requirements: FREQ.1 to FREQ.5. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 32 - - - - Use Case Basic Stimulus and Responses - 1. User authenticates in the platform - 2. User is provided access to platform interfaces for service deployment - 3. User deploy services - 4. Platform checks compatibility and any technical issues arising from the - UC12. Deploy new service - Services 5. If approved the service is ready to be used and managed in the - platform - o If deployment is not approved, platform shows error message and - returns to step 1. - 6. End of deployment - - - -2.7.2 Use Case: Manage Services -ID: UC13 -Refines: GOAL 5: User’s experience is enhanced by the provision of value-added services -Pre-condition: User successfully authenticates in the platform -Actors: Data Services Providers -Rationale: Manage services provides the functions for updating, maintaining and accessing -services as well as tracking their usage by users. Ideally the owners of the services should be the -only authorised user to manage them, and other authorised users can track the usage of the -services in the platform. In case of updates the platform must log in the database update details -and send to service providers a response indicating the status of the update. The platform should -also ensure update errors are not propagated nor affect the health of other services provided in the -platform, and should keep an audit trail of all actions to enable rollback. Data services usage -tracking includes performing queries on the data management data to generate result sets, and -producing reports from these result sets. All user’s information provided to service providers must -follow regulations of data protection and the user’s defined rules for their data use. -Specialised Use Cases: The Use Case Manage Services is distinguished into two specialised -Use Cases: “User manages services (UC13.1)” and “User tracks service usage (UC13.2)”. -Subordinated Use Cases: “Transmit Data (UC5)” -Refines into requirements: FREQ.27 to FREQ.25. - - - Specialised Use Cases Basic Interactions and Responses - 1. Platform provides user with an interface for services management - 2. User chooses to edit or delete services - 3. If edit, user revise service information (access-control, - commercial models, parameters) and deployment; - If delete, user selects services to be removed / disabled - UC13.1. User manages 4. User confirms action - services 5. Platform quickly process user’s request - 6. Platform confirms execution of request - o If valid request, platform acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 7. End of services management - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 33 - - - 1. Platform provides user with an interface for services management - 2. User chooses to visualise usage information of a service - 3. Platform quickly process user’s request for data usage - UC13.2. User tracks information - services usage 4. Platform provides user with statistical information about services - usage and data users anonymised information - 5. End of data services tracking. - - - -2.7.3 Use Case: Utilise Data Services -ID: UC2 -Refines: SUB-GOAL 1 - City data is collected in an intelligent manner -Pre-condition: User is authenticated in the platform -Actors: City data publisher -Rationale: Data Consumers can register in the platform and request approval to consume city data via GUI -or APIs. They provide valid registration details (to be defined) and wait for platform to confirm their -registration. -Subordinated Use Cases: “Commercialise Data Services (UC8)” -Refines into requirements: FREQ 6 to FREQ.26. - - - Specialised Use Cases Basic Interactions and Responses - 1. Users / Machines select data service to be utilised - 2. Users / Machines are redirected to authentication mechanism in - UC2.1. User utilises data case of registration is needed for the particular service - services o If authentication is successful, users are provided with - requested data streams - 3. Users are provided with requested service either via API or GUI - - -2.7.4 Functional Requirements - - Req. ID UC. ID Description Priority Domain - Provide stable and well-defined interfaces to ensure interoperability - Societal Needs, - FREQ.84 UC12 between the platform, services and the applications provided by Should Platform - services providers - Ensure the interfaces of the architecture are open to reduce entry Societal Needs, - FREQ.85 UC12 Should - barriers and integration issues Platform - Provide multi-purposed and network intelligent interfaces to Societal Needs, - FREQ.86 UC12 Must - providers and consumers of services Platform - Provide service providers mechanisms to define the terms and Societal Needs, - FREQ.87 UC12 Must - conditions of platform services deployment Platform - Business Needs, - FREQ.88 UC13 Provide statistical information of user’s feedback on service usage Must Platform - Allow users to use services to manipulate city data (e.g. Create Societal Needs, - FREQ.89 UC10 Must - mash ups, integrate) Platform - Allow users to provide feedback on usability, and quality of data Societal Needs, - FREQ.90 UC14 Must - and services provided by the platform Platform - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 34 - - - -3. Non-functional Requirements -3.1 Run-time Quality Requirements -3.1.1 Scalability Requirements - -Rationale: The ability of the system to execute its task within its expected performance profile and -to handle on-demand increased processing volumes of data and service requests -Drivers: Provide ready access to all data that underpins decision making processes in smart cities, -and accommodate users and data usage patterns -Refines into requirements: NFREQ.1, NFREQ.2, NFREQ.3, NFREQ.4 -Relevance: Urban platforms have ambitious performance requirements. Such platforms must cope -with user’s demand for data and services, capture real-time data that will be catalysed by a myriad -of sensors. The demand for urban platform is very likely to significantly increase over time. It is -very difficult to have clear performance characteristics due to the ubiquity, heterogeneity high -connectivity of devices and end users. - - SCALABILITY MEASURES - Actions Capture the performance requirements - Create service level agreements - Predict scalability using software simulation - Analyse the performance of the platform overtime - Conduct practical testing - Strategy Prioritize service and data requests - Distribute processing over time - Scale up or scale out as necessary - Reuse resources and results Partition and parallelize - Constantly monitor Quality of Service at runtime - - - -3.1.2 Availability and Reliability Requirements - -Rationale: The ability of the system to be fully or partly operational as and when required and to -effectively handle failures that could affect system availability -Drivers: Build a reliable foundation for “on demand” exploitation of data -Refines into requirements: NFREQ.5, NFREQ.6, NFREQ.7, NFREQ.8 -Relevance: Any system that has complex or extended availability requirements, complex recovery -processes, or a high profile (e.g., is visible to the public) - - AVAILABILITY AND RELIABILITY MEASURES - Actions Capture the availability requirements Produce the availability schedule - Estimate platform availability Estimate functional availability Assess against the - requirements Rework the architecture - Strategy Adopt fault-tolerant hardware - Use reliable infrastructure database - Log transactions - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 35 - - - Develop adaptive software to cope and recover from faults - Design and test for failure - Deploy load balancing - Identify suitable backup and disaster recovery solution - - -3.1.3 Trust Requirements - -Rationale: A quality related to the user’s belief in the reliability, integrity and ability of the functional -behaviour of the platform -Drivers: Gain understanding of what influences user’s experience while interacting with services -provided -Requirements: NFREQ.16, NFREQ.17 -Relevance: Relevant to the systems that share and collect information that may raise public -concern. In some cases, trust has to do with the reliability of data and their providers, whereas in -other cases trust can be associated with the security and privacy of the technology that was -deployed. Trust affects the reputation of the platform besides its dissemination and maturity on the -market. - -TRUST MEASURES - Capture trust requirements - Perform risk analysis so measures can be implemented -Actions Explore the vulnerability aspects of city data - Check whether extensibility requirements impact on trust - Define a trust model - Adopt trust model - Deploy monitoring capabilities and assess its impact on scalability - Manage the data in a way that ensures its compliance with data -Strategy protection regulations - Implement tampering and data misuse detection - Make use of Cryptography when necessary - Allow Federation of trust between platforms - - - -3.1.4 Security Requirements - -Rationale: Ability of the system to enforce the intended confidentiality, trust, integrity and service -and data access policies, and to detect and recover from failure in these security mechanisms. -Drivers: Manage the data and services in a way that ensures its integrity, and compliance with -data protection regulations -Relevance: Relevant to the systems that share and collect information that may raise public -concern. Urban platforms may become a valuable target for attackers which can potentially leave -huge swathes of information exposed. It could potentially undermine trust in the government and -damage its reputation. -Refines into requirements: NFREQ.9 to NFREQ.15 - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 36 - - - SECURITY MEASURES - Elicit the security requirements - Cross-check security requirements’ impact on services offered by different - providers - Verify security impact on service composition - Conduct risk analysis and impacts of security breach - Use scalable and efficient user authentication components - Use authentication and authorization components to secure interfacing with - Actions external services - Address aspects of data collection, storage and distribution - Address aspects of service and communication security among devices - Identify security requirements of the physical infrastructure - Balance and prioritize scalability (performance) and security requirements - Evaluate trade-offs between privacy considerations and preventing abuse. (While - anonymous access guarantees privacy of users, traceability of users due to abuse - of the service is not possible.) - Toughen user’s functional components - Authenticate subjects - Define and enforce access policies - Secure communication infrastructures - Secure interfaces with external systems and services - Strategy - Secure databases - Check data integrity for critical services - User digital certificates and encryption where necessary - Secure monetary transactions - Secure user’s personal information - - -3.1.5 Privacy Requirements - -Rationale: Ability of the system to ensure that the collection and transmission of personal data is -minimized and handled in accordance with user’s expectation and regulations. -Drivers: Protect the vulnerability aspects volunteered citizen’s data -Requirements: NFREQ.18, NFREQ.19, NFREQ.20, NFREQ.21, NFREQ.22 -Relevance: Ensuring users privacy is protected positively influences user’s experience, -acceptance and continuous use of the platform. Besides other factors, the reputation of the -platform depends on how well user’s information is secure and preserved. - - PRIVACY MEASURES - Capture trust requirements - Perform risk analysis so measures can be implemented - Actions Explore the vulnerability aspects of city data - Check whether extensibility requirements impact on trust - Define a trust model - Allow users to interact with the platform anonymously in given - circumstances - Manage users identification securely - Strategy Anonymize data whenever necessary to comply with data regulations - Cryptograph personal identifiers during data transmission when that is - necessary - Avoid unauthorized access to implicit information (e.g. location) - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 37 - - - Verify the impact of security, trust and scalability requirements trade-offs - on privacy - Allow the user to control how personal information is used - Empower user to control the data disclosure mechanism - - - -3.2 Non Run-time Quality Requirements -3.2.1 Evolvability Requirements - -Rationale: The ability of the platform to withstand and easily adapt when new requirements and -changes is introduced. -Drivers: Ensure the platform is able to accommodate additional functionality and emerging -technologies at later stage at a fair and transparent cost -Refines into requirements: NFREQ.23 -Relevance: Important for longer- lived and more widely used systems. Urban platforms are -expected to be highly evolvable in order to accommodate future emerging technologies and avoid -interoperability issues. - - EVOLVABILITY MEASURES - Actions Characterize and assess the evolution needs - Consider the evolution trade-offs - Balance and negotiate potential conflicting requirements emerging from - the evolution capability. - Strategy Adopt standards and open interfaces - Design loose-coupled components - Preserve the platform resilience - - - -3.2.2 Extensibility Requirements - -Rationale: The flexibility of the system to allow services and functionality to be extended and -augmented by service providers in order to increase value of services to both platform providers -and end-users. The extension of the platform services is determined by the Platform Openness -strategy. -Drivers: Stakeholders can extend the services provided by the urban platform, so that -partnerships can be built to deliver holistic and interoperable solutions. Identify integrated -approaches to design and service delivery which ensures that services fit together and that -synergies can be exploited. -Refines into requirements: NFREQ.24, NFREQ.25 -Relevance: Important for longer- lived and more widely used systems. Urban platforms are -expected to be highly evolvable in order to accommodate future emerging technologies and avoid -interoperability issues. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 38 - - - EXTENSIBILITY - Actions Characterize and assess the extensibility needs - Trace the impacts and cost of extensions - Negotiate potential conflicting requirements emerging from adding extensions - in the platform - Strategy Develop a modular architecture with standard interfaces that reduces entry - barriers due to increased transparency and integration - Share technical information about interfaces will help service providers in - targeting opportunities around the platform - - - -3.3 List of Non-Functional Requirements - -ID # Description Concern Priority Domain - - Platform, -NFREQ.1 Support different service level agreements (SLA) Scalability Should - Infrastructure - Platform, -NFREQ.2 Process services and events on a set of distributed nodes Scalability Should - Infrastructure - Platform, -NFREQ.3 Continuously monitor Quality of Service at runtime Scalability Should - Infrastructure - Platform, -NFREQ.4 Balance its load at runtime Scalability Should - Infrastructure - Platform, -NFREQ.5 Provide high availability Availability Should - Infrastructure - Platform, -NFREQ.6 Guarantee infrastructure availability Availability Should - Infrastructure - Platform, -NFREQ.7 Ensure network availability Availability Should - Infrastructure - Platform, -NFREQ.8 Be able to perform self-healing Availability Should - Infrastructure - City Data, -NFREQ.9 Expose data and services to authorized users Security Must - Platform - City Data, -NFREQ.10 Ensure services are always accessible to entitled users Security Must - Platform - -NFREQ.11 Ensure Data Freshness Security Must Platform - -NFREQ.12 Support access control mechanisms Security Must Platform - - Platform, -NFREQ.13 Have security mechanisms to protect data transmission Security Should - Infrastructure - Platform, -NFREQ.14 Make it difficult to spy on communicated messages Security Should - Infrastructure - Platform, -NFREQ.15 Be able to perform to detect threats at runtime Security Should - Infrastructure - Provide trusted and secure communication and information Platform, -NFREQ.16 Trust Should - management Infrastructure - -NFREQ.17 The platform infrastructure and services shall be trustable Trust Should Infrastructure - - Societal Needs, -NFREQ.18 Allow users to use free services anonymously Privacy Should - Platform - Societal Needs, -NFREQ.19 Allow people to use free services anonymously Privacy Should - Platform - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 39 - - - Allow users to control which data they are willing to provide Societal Needs, -NFREQ.20 Privacy Should - and how their data should be used Platform - -NFREQ.21 Keep users access-control rights/ policies secured. Privacy Should Platform - - Provide privacy protection for users interacting with the Societal Needs, -NFREQ.22 Privacy Should - platform Platform - Platform, -NFREQ.23 Provide communication confidentiality Privacy Should - Infrastructure - Societal Needs, -NFREQ.24 Be extensible for future technologies. Evolvability Must City Data, Platform, - Infrastructure - Platform, -NFREQ.26 Provide standard interfaces for service providers Extensibility Should - Business Needs - -NFREQ.26 Be able to provide services in an interoperable manner Extensibility Should Platform - - - - -4. Other Requirements -4.1 Minimal Descriptive Metadata Required from Data Provider - -Minimal Descriptive Metadata Required from Data Provider - - Attribute Required Definition Notes Examples - -Database Name Y A name given to the resource - -Author Name of a person or body associated with - Y - the creation of the resource - -Maintainer Name of the entity responsible for making - Y - the resource available - -Date Created Y Date the dataset was created - - -Date Modified Y Date the dataset was modified - - -Last Revision Y Date the dataset was last revised - - -Update Frequency Y Frequency of data maintenance - - -Mode of Release Y Open/Proprietary - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 40 - - -4.2 Urban Platform Supported Data Formats - - -MIME type Description Extensions Level - -application/json JavaScript Object Notation json supported - -application/xml eXtensible markup language file xml supported - -text/csv Comma separated values file csv supported - - - - -5. Conclusion & Forward Plans -This document represents the first set of requirements specification for urban platform. Future -activities will collaboratively assess, resolve requirements conflicts, prioritize, and validate the -requirements of the urban platform. Ultimately, this document will become a complete final -requirements specification document to guide and speed up the development open platform for -cities. Table below shows the milestones, deliverables and engagement activities completed and -yet to be completed by the Demand Side Engagement Stream. - - - - Milestones, deliverables and engagement activity Forecast date Status - Stakeholders invitation Late Completed - September/2015 - Online Workshop 1: Project Scope and Outcomes 20/11/2015 Completed - Online meeting held with city members to communicate goals, - expected outcomes, time commitment and required actions. - Start of Collaborative Requirements Engineering Process 20/11/2015 Completed - Participants reviewed the first draft of urban platform requirements, - provided comments, and suggested new requirements and changes. - Online Workshop 2: Review and Refine Requirements 08/12/2015 Completed - Participants had a conference call to review the requirements - specification document and provide their comments. - Deliverable: Requirements Specification Document v2.1 shared 05/01/2016 Completed - across the Working Streams for consultation - Online Workshop 3: Review Requirements Specification 12/01/2016 Completed - Document and discussion of Letter of Intent - Participants had a conference call to review the requirements - specification document and provide their comments, and discussed - the Letter of Intent to be signed by Mayor/Equiv of EU cities to - commit to application of common U.P. approach. - Urban Platforms Workshop in Brussels 19/01/2016 Completed - Participants provided feedback on the requirements specification - document and recommended changes in the document - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 41 - - - Deliverable: Updated Requirements Specification Document v2.2 29/01/2016 Completed - shared with Industry and Standardisation Working Streams - Deliverable: Online release of the Requirements Specification Early - Document and open calls for wider consultation on the Requirements February/2016 - - Online Workshop 4: Engagement with Industry Side Early - Capture industry feedback on the Requirements Specification February/2016 - Document to assess its suitability to drive the development of the first - draft reference architecture. - Online Workshop 5: Review Requirements Specification Mid - Document v2.2 and discuss requirement prioritization and February/2016 - balancing - Collectively determine which candidate requirements of urban - platforms should be prioritized as high importance to all the cities - (use of Value Oriented Prioritization Method). Requirements are also - prioritized to minimize risk during development of urban platforms so - that the most important requirements are made explicit and - considered in the reference architecture. - Capture of case studies from cities intended to support requirements Mid March/2016 - validation (City of Porto), and learning and confidence building - amongst cities. diff --git a/files/extracted_requirements.txt b/files/extracted_requirements.txt deleted file mode 100644 index 80f9fc2..0000000 --- a/files/extracted_requirements.txt +++ /dev/null @@ -1,2455 +0,0 @@ -Software Requirements Specification for Urban Platforms (EIP Project) Page i - - - - - Requirements Specification - For - - Urban Platforms - Version 2.2 - (Consultation Stage) - - - - - Prepared by Demand-Side Engagement Stream - - - - -European Innovation Partnership for Smart Cities & Communities (EIP_SCC) -Integrated infrastructure Action Cluster – Urban Platform - Software Requirements Specification for Urban Platforms (EIP Project) Page ii - - - - -Requirements Specification Life-Cycle -Revision history - - Author Date Reason for change Phase Revision - - Dr. Larissa Romualdo-Suzuki 4/12/2015 - Initial Draft Version 0 - Outline - - Dr. Larissa Romualdo-Suzuki 21/12/2015 Working update Core Demand- Version 1 - Side Team - Consultation - - Dr. Larissa Romualdo-Suzuki 05/01/2016 Incorporate cities Development Version 2.1 - comments - - Dr. Larissa Romualdo-Suzuki 29/01/2016 Incorporate suggestions Development Version 2.2 - from Urban Platforms - Workshop held in Brussels - on the 19/01/16. - - - - -Requirements Specification sign-off: - - Name: Andrew Collinge - Role: Demand Side Engagement Lead Date - Organisation: Greater London Authority 11 January 2016 - - - - -Acknowledgement: - -This document has been developed in consultation with a representative group of European cities from -different locations, size, and level of development. Cities include: London, Amsterdam, Barcelona, -Syracuse, Berlin, Gent, Valencia, Murcia, Derry, Copenhagen, Scottish cities, Porto, Riga. We wish to -recognize the commitment, support and ongoing contribution from these cities to develop a common -statement of needs. - Software Requirements Specification for Urban Platforms (EIP Project) Page iii - - - - -Table of Contents -Table of Contents ...................................................................................................................... iii -1. Introduction ............................................................................................................................1 - 1.1 Purpose and Scope of this Document.................................................................................. 1 - 1.2 Context ..................................................................................................................................... 1 - 1.3 Intended Audience .................................................................................................................. 2 - 1.4 Definitions, Standards, and Framework ............................................................................... 3 - 1.5 How to Use this Document .................................................................................................... 2 - 1.6 Product Scope and Perspective ............................................................................................ 5 - 1.7 The Urban Platform Development Stack ............................................................................. 7 - 1.8 User Classes, Characteristics, and User Access ............................................................... 8 - 1.9 User Documentation ............................................................................................................. 10 - 1.10 Design and Implementation Constraints ............................................................................ 10 - 1.11 Assumptions, Alignment with other Action Clusters and Policies ................................... 10 -2. Urban Platform Value Proposition, Use Cases and Functional Requirements ..12 - 2.1 From Value Proposition to Platform Specifications .......................................................... 12 - 2.2 Overall Goal: “City data is exploited to its full potential” .................................................. 13 - 2.3 SUB-GOAL 1: City data is collected in an intelligent manner ......................................... 15 - 2.4 SUB-GOAL 2: City data is managed in a safe and intelligent manner .......................... 20 - 2.5 SUB-GOAL 3: City data is orchestrated in a marketplace ............................................... 23 - 2.6 SUB-GOAL 4: City data is offered in an accessible manner ........................................... 28 - 2.7 SUB-GOAL 5: User’s experience is enhanced by the provision of value-added - services............................................................................................................................................... 31 -3. Non-functional Requirements..........................................................................................34 - 3.1 Run-time Quality Requirements .......................................................................................... 34 - 3.2 Non Run-time Quality Requirements.................................................................................. 37 - 3.3 List of Non-Functional Requirements ................................................................................. 38 -4. Other Requirements ...........................................................................................................39 - 4.1 Minimal Descriptive Metadata Required from Data Provider .......................................... 39 - 4.2 Urban Platform Supported Data Formats .......................................................................... 40 -5. Conclusion & Forward Plans ...........................................................................................40 -Appendix ......................................................................................... Error! Bookmark not defined. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 1 - - - - -1. Introduction -1.1 Purpose and Scope of this Document -The purpose of this document is to provide a common core set of city-needs-led requirements, co- -developed and validated by EU cities, to support the acquisition of “Urban Platforms” by EU cities. -The requirements are made openly available. The ambition is that adoption of these requirements -by EU cities will lead to reduced pre-procurement times, increased confidence in platform designs, -greater levels of collaboration (particularly amongst smaller cities), innovation in business models, -more affordable solutions, and a more secure basis for industry to apply its innovations. This will -lead to accelerated adoption of urban platforms by EU cities, so that they can exploit the potential -of the growing volumes of city data, and improve the services to and outcomes for their residents, -business community and visitors. - -This document provides a primary input to the EIP_SCC Urban Platform Supply-Side and -Standardisation workstreams as a reference for the development of further technical and -operational materials. The requirements illustrate the purpose and complete declaration for the -development of system, as well as system constraints, interface and interactions with other -external applications. - -The scope of this document is limited to common services data platform requirements, and issues -related to the management of city data within an Urban Platform. This considers the full life-cycle -of city data, including: the maintenance of the inventory of existing city data; development of -functional requirements for common services data platform from a city-needs perspective; -identification of metadata and format standards for city data; and identification of policy issues -related to the data platform. It includes in the scope the functionality of software and systems used -to handle existing data catalogues (e.g., open data portals). - -Commercial off-the-shelf (COTS) software or hardware, as well as databases, which manage data -that are not part of Urban Platform’s collecting responsibility (e.g., mobile applications, proprietary -Internet of Things, proprietary data catalogues) are outside the scope of these requirements. - - -1.2 Context -The initial EIP_SCC Demand-Side survey1 clearly indicated that the vast majority of EU cities did -not have an urban platform; that there were a number of significant barriers to adoption (including -funding, capabilities, and the willingness to work across service ‘silos’); that there was limited -collaboration in this area; and as a result most cities were ‘sitting on the fence’. The opportunity -clearly is to unblock this. Notably because urban platforms provide a vital foundation for smart city -infrastructure and service improvements. So their adoption is considered a strategic priority by the -EIP_SCC, and by a growing number of cities. - -There are a number of clear trends that any city, and thus these requirements, must recognise: - • The astounding increase in volumes of city data, driven notably by IoT/sensor/M2M - implementations, and social media - • The pressure to improve and make open data from public (and private) sources; - -1 - Survey performed Q1’15 and reported on the EIP_SCC Marketplace site - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 2 - - - • The continued reality of austerity that drives cities towards transformative solutions - -Cities presently typically have a mix and variety of qualities of legacy systems, with various -degrees of technical ‘platform or platforms’ in place. It is recognised that the roadmap a city might -follow to transition to a more aligned and functional state will differ in many ways; and that the end -point does not envisage ‘one (physical) platform’. However cities will benefit from adopting more -consistent conceptual, and logical architectures, which will lead to more efficient and effective -physical operations, and greater benefits in terms of exploiting city data. - -Furthermore it must be recognised that cities will continue to wrestle with capacity challenges to -improve quality, availability, governance, and appropriate monetisation of city data. - - -1.3 Intended Audience -This document is co-developed by and initially intended for the growing number of city members of -the Demand-Side Engagement Stream of the EIP_SCC Urban Platform initiative. Its formal release -as a common statement of requirements by representative EU cities will form the basis for -commitment by EU cities to the initiative by way of a Letter of Intent (LoI). The spirit of which seeks -to ensure broad use of these requirements by EU cities (and recognition of deviations) to help -open the smart city market. - -It is also intended to be used by Industry in setting an agreed city-needs-led set of requirements -which can become more consistent and common across EU cities. This will lead to advantage for -cities and Industry. - - -1.4 How to Use this Document -Cities are encouraged to use this document as a guiding reference for the development of their -own platform specifications. - -The following steps offer a logical means to extract the full value of this document: - - - 1. Identify the priority service outcomes that the city seeks to achieve to steer the overall - prioritisation of urban platform development - 2. Perform a city data mapping exercise to develop a picture of the data landscape, including: - sources, volume, variety, temporal factors and sensitivity, data licensing and ownership - policies. - 3. Map out existing ICT system resources across the city in order to identify those resources - with the greatest potential for reuse, identify gaps and provide the foundation for a data - strategy and technology plan to fill them. - 4. Identify the requirements within this document that best suit the city needs - alongside any - other emerging requirements – and refine and prioritise them in accordance to their data - strategy defined in step 1 and 2. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 3 - - - 5. Cities can create their own specification document in their own language and refer to the - requirements of this document by using the unique identifier of the requirements (which - signals to providers the similarities and deviations from the core set) - 6. Determine what ICT infrastructure and sets of requirements are needed citywide to support - the urban platform. - - - - -1.5 Definitions, Standards, and Framework -To ensure a robust basis for market adoption we have set in place a number of foundations. Firstly, -a common definition of terms. Secondly, the adoption of some core accepted standards. Thirdly, -an underlying model to develop requirements. - -1.5.1 Working Definitions -For the purpose of this document we assume the working definitions developed within and shared -across all working groups of the Urban Platform initiative - -• An ‘Urban Platform’… - - o Implements a logical architecture/content/design that brings together (integrates) data flows - within and across city systems and exploits modern technologies (sensors, cloud services, - mobile devices, analytics, social media etc); - o Provides the building blocks that enable cities to rapidly shift from fragmented operations to - include predictive effective operations, and novel ways of engaging and serving city - stakeholders; - o Transforms, in a way that is tangible and measurable, outcomes at local level (e.g. increase - energy efficiency, reduce traffic congestion and emissions, create (digital) innovation - ecosystems, efficient city operations for administrations and services). - -• ‘City Data’ is… - - o Data that is held by any organisation - government, public sector, private sector or not-for- - profit - which is providing a service or utility, or is occupying part of the city in a way that can - be said to have a bearing on local populations and the functioning of that place; - o Consists of varied characteristics such as static, near-real time or in the future, real time, - descriptive or operational. - o It will be to a greater extent generated by individual citizens and this too (with due - consideration to privacy and a strong trust framework) can be considered city data. - - -For the purpose of this report, the following terms and definitions4 apply. - - o Open data: non-privacy-restricted and non-confidential data. Produced with either public or - private resource and is made available without any restrictions on its usage or distribution. - o Private data: restricted and/or licensed data including permission, charging, privacy, - publication and distribution. Produced with either public or private resource. - o Commercial data: restricted and/or licensed data including permission, charging, privacy, - publication and distribution. Produced with either public or private resource. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 4 - - - o Sensory data: open and/or restricted data collected by different sensors, actuators and - devices owned by public and private sector, and citizens. Sensory data is usually diverse in - nature and architectural features, mostly location and time dependent, and present different - levels of quality. - -These types of city data are produced within the public and private sectors and crowdsourcing -initiatives, and have their respective suppliers and distribution strategies (e.g. reports, APIs, -datasets). - - o Public sector data: restricted data relating to location, national security, commercial - sensitivity and privacy. Data produced, collected or funded by the public sector. - o Private sector data: data produced, collected or funded by the private sector, which can be - open, private and commercial data. In case of private data, the restrictions of usage and - distribution are decided by the individual businesses. - o Crowd-sourced data: data provided, collected and distributed by humans through the use - of digital technologies and social media. - - -1.5.2 Use of Standards - -• We seek to ensure Urban Platforms in EU cities are designed with regard to international best - practices for data repositories. -• As new guidance and standards are developed the three streams of the EIP_SCC2 Urban - Platform initiative will review them for broader use. (Note: the SCC03 EC project and - consortium is explicitly linked within this initiative and will act on Urban Platform standards). -• For now, this document assumes a modified IEEE 830-19983 layout for software requirements - specification, as the basis to capture a high-level statement of the urban platform’s - requirements. - - -1.5.3 The Organising Framework - -This document uses the dynamic business models framework4 illustrated in Figure 1 to organise -and guide the definition of goals, the development of use cases, and definition of requirements -during the life cycle of urban platforms. Future iterations of this dynamic design process are -Procurement, Roll-Out and Market phases. Consequently, this document may include, alter, -prioritize, extend or remove specifications as they become known. - -The principal goal addressed is that of “exploiting the value of city data through urban -platforms”. From this we have developed five common sub-goals. Each includes: a description; -rationale; drivers; and actions. These sub-goals are elaborated through a total of 14 use cases and -developed into a total of 116 functional and non-functional requirements as illustrated in Figure 2. -The use of the requirements by Industry will be subject to a further update. - - - - -2 - EIP Integrated infrastructure Urban Platform Governance Model, Version 3, 28th July 2015. -3 - IEEE 830-1998 Standard for Software Requirements Specifications, 1998. -4 - Framework for Data Infrastructures Design from “Data as Infrastructure for Smart Cities”, Suzuki-LCSR, -PhD Thesis, UCL, 2015. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 5 - - - - - Figure 1. Organising Framework4. - - - - - Sub-Goal 1: City data is provided - Use Cases: 3 Requirements: 34 - in a harmonised way - value of city data through an - Principle Goal: Exploit the - - - - - Sub-Goal 2: City data is managed - Use Cases: 3 Requirements: 36 - in a safe and intelligent manner - urban platform - - - - - Sub-Goal 3: City data is - Use Cases: 2 Requirements: 12 - orchestrated in a market place - - - - - Sub-Goal 4: City data is offered in - Use Cases: 3 Requirements: 11 - an accessible manner - - - - - Sub-Goal 5: User’s experience is - enhanced by the provision of Use Cases: 3 Requirements: 8 - value-added services - - - - Figure 2 Structure of this document. - - - - -1.6 Product Scope and Perspective -The EIP Project’s urban platform is an open common architecture which serves for city data -collection, management and distribution. An urban platform is intended to support the widespread -exploitation of city data by humans and machines in the urban environment. Figure 3 illustrates a -holistic high level overview of the urban platform the EIP project intends to deliver. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 6 - - - -The reference architecture for urban platforms should: - -• Cater for interoperability between urban infrastructures -• Enable replicability of the solutions/platforms city to city -• Scale without technical constraints and excessive cost increase -• Provide open APIs and SDKs -• Enable Real Time capabilities -• Support implementation of functional and technical capabilities - - - - - Figure 3. High level overview of the urban platform (currently approved EC DG CNECT). - - -The current urban platform market is nascent. Many software vendors offer such a platform, though -many requirements and expectations of the stakeholders of city data are not (fully) addressed. As -a result, current urban platforms are often more costly to design and maintain, less reusable and -often not interoperable platform-to-platform, and susceptible to information fragmentation and -overload. - -The urban platform which the EIP initiative intends to design takes a step beyond the platforms -currently on the market by ensuring the requirements are fully founded on a co-created and -common set of representative city needs, from which it solicits suitable industry input, and an open -and managed collaboration between industry, cities and communities, and others, in order to take -into account their needs and concerns. To do this, it is necessary to take a technology agnostic -approach to design an open and common reference architecture for urban platforms. This platform -must ensure data is collected and sustained in accordance with well-stablished standards, -managed in a robust manner so that it can handle high level supply and demand of data, and -distributed across different value chains, systems and stakeholders. The ability to handle high level -of city data supply and demand while being user secure and accessible enough for city-wide -exploitation of data is one of many keys to the success of urban platforms. This is central to the -design and implementation of urban platforms. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 7 - - -1.7 The Urban Platform Development Stack -The requirements specification of this document is based on the Urban Platform Development -Stack illustrated in Figure 4. The stack is composed by five domains (represented as layers in the -stack) necessary to fully implement an urban platform software suite as shown in Table 1. Each -domain comprehends a set of requirements necessary to the design of a common and open urban -data service platform. The elicited requirements are used to define a technical architecture which is -simple enough to be comprehensible at least at a high level of abstraction. The platform should be -conceptually decomposable into its major subsystems, the platform’s functionality reused by many -services and external applications should be identifiable, and interactions between the platform -and services, data providers and data consumers should be well defined and explicit. - -The first layer of the stack “Societal needs” concerns to outcomes we strive for within a portfolio of -city service domains. An urban platform should recognise societal needs and wants as the starting -point for city data service offering. Ultimately, an urban platform aims to provide tailor made and -compelling engaging services for the users. The Services and Business models layers concerns -with delivering data services which carefully targets the needs and expectations of the different -users of the urban platform, and explore use cases and commercial models where data is used to -deliver different forms of value. The city data layer concerns with the mechanisms necessary to -transform urban platforms into a foundation for widespread exploitation of data, including handling -data architectural features, data usability, semantics and quality aspects. The urban platform layer -concerns to the technology foundation to configure, share, and interpret exponentially increasing -volumes city data and services. Finally, the Infrastructure layer concerns with the base level -connectivity that supports the platform to be scalable and reliable in the long run. - - - - STACK OUTPUT - - Requirements to deliver new digital services that will address the - Societal Needs societal needs of cities in a positive manner that relates to political - narratives. - - - Services & Requirements to new profitable business models and the development - Business Models of an increase range of new and engaging services in the smart cities. - - - - City Data Requirements to provide all city data stakeholders ready access and - delivery of all city data that unpins the decision making process in smart - cities. - - - Urban Platform Requirements to put in place applications together to build a foundation - for the widespread exploitation of data. - - - Infrastructure Requirements to deliver the backbone infrastructure that will be used to - capture the opportunities of digital technology and data to enable - transformation. - - - Figure 4. Urban Platform Development Stack. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 8 - - - Table 1. Urban Platform Development Stack - Layer Rationale - Societal Needs - Accessible services and data necessary to solve social problems - and drive innovation; - - Parameters that influence user’s experience while interacting with - services (e.g. usability, feeling of security and trust); - Services and Business - Tailor-made data services which careful targets the needs of users - Models and businesses; - - New potential and cost-effective beneficial services that could be - rolled out across cities of different sizes; - - Use cases where data is used to deliver different forms of value. - City Data - Data architectural features (e.g. volume, variety, temporal factors - and sensitivity); - - Data licensing, policies and regulations to exploit data to full effect; - - Minimum metadata requirements; - - Data usability and reusability aspects of humans and machines. - Urban Platform - Holistic and interoperable solutions; - - Integrated approaches which ensures that services fit together and - that synergies can be exploited; - - Data management mechanisms to ensure data integrity and - compliance with data protection regulations - - Extension capabilities to accommodate additional functionality at - later stage at a fair and transparent cost. - - - -1.8 User Classes, Characteristics, and User Access -The users of the Urban Platform include end-users, such as the general Public, public and private -organisations; data providers; service providers; and the platform providers who will be -working with the providers of city data and services, and managing the content, defining policies -and regulations of the platform. A crucial feature of an urban platform is the provision of the -various access levels required by the different types of users. Particular uses need different -access levels to some data than the general public. Data publishers will require access to the -Urban Platform in order to ingest, administer, manage, preserve and access their resources. This -will require multiple levels of access to city data and its respective metadata. Table 2 provides a -description of each class of users. - - - Table 2. Actors - User Class Rationale - Platform - Maintains the ecosystem of data, services and users; - Provider - Defines standards, licenses and regulations and provides terms and conditions - for platform usage and the commercial exploitation of data and services; - - Decides who are allowed to join the value network of data and services - providers; - City Data - Publishes open and proprietary data into the platform; - Publisher - Manages and maintain resources in the platform accordingly to terms and - conditions. - Data - Deploys open and commercial data services into the platform (e.g. data - Services visualisation, data cleansing, data integration tools); - Provider - Manages and maintain resources in the platform accordingly to terms and - conditions. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 9 - - - City Data - Consumes open and proprietary data provided in the platform; - Consumer - Uses open and commercial data services provided in the platform; - - Provides feedback on data and services provision; - - -1.8.1 End-User Access - -City data consumers will need to access and use the city data residing in the Urban Platform. End- -users will be able to search metadata and full text within datasets (when available), and obtain city -data in open formats readily available to both humans and machines such as CSV, XML, JSON. -Some end-users may require different access rights to city data. The 2 major end-user groups that -have been identified are: - -• Open data users, including both national and international users (humans and machines). - Open access to some city data may be restricted by licensing terms (e.g. commercial data), - embargo periods, copyright, etc. - -• Private data users, which need to use the Urban Platform to obtain commercial city data. Data - access is available via data subscriptions or when purchase requirements and licenses are - waived by the data provider. - - -1.8.2 City Data Publisher Access - -A broad data provider level access is needed for stakeholders (humans and machines) working -with the urban platform and their respective data in it. Basically, data publishers will carry out the -following activities: -• Data publication access, available to publishers adding new data and metadata, checking the - quality of datasets, manipulating data, performing format conversions, defining data-access - level, tariff for consumption when applicable, and licences. - -• Data maintenance access, for publishers reviewing or editing appropriate data and metadata in - the urban platform. Data publishers can view data and add to or edit metadata without - changing the data itself. They should be provided with access to feedback from users to - investigate problem in their resources (e.g. missing data, inconsistent metadata), and statistical - information about how their resources are used by users. - - -1.8.3 Data Services Provider Access - -This is the second most restrictive access level providing rights to deploy services in the platform. -Basically, data service providers will carry out the following activities: - -• Data services deployment access, available to service providers adding new mechanisms or - integrating new applications, testing and validating integration, defining data-access level and - tariff for service usage. - -• Data services maintenance access, for services providers reviewing, extending or editing - applications in the urban platform. Data services providers can view their services deployed - and add to or edit access level and tariff without having to deploy the services again. They - should be provided with access to feedback from users to investigate problem in their services - (e.g. bugs, scalability issues), and statistical information about how their services are used by - users. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 10 - - -1.8.4 Platform Provider Access - -This is the most restrictive access level providing ultimate rights to the system and is required for -its management, development, and assigning appropriate rights to data and services providers. -Policies and regulations, license agreements are also defined by the provider of the urban -platform. Platform providers should be provided with the means to follow up on civic engagement -(e.g. feedback, request for city data) and on the provision of city data and services. - - -1.9 User Documentation -• City Data Consumers: Provide license terms and conditions associated to consuming - data and services provided in the platform, documentation of APIs and guide to - discover city data in the platform. -• City Data Providers: Provide data publication documentation describing the minimum - metadata requirements, formats accepted, step-by-step guide to publish accurate city - data. -• Data Service Providers: Disclosure technical and architecture blueprint details in - order share and outsource expertise, and partnerships, and integrate supporting - partners’ solutions into the platform itself. - - -1.10 Design and Implementation Constraints -1.10.1 Design Constraints - -• Lack of standards agreement for metadata representation. -• City data found in existing data catalogues may require special consideration concerning the - type of formats and datasets that must be stored within the platform. -• Requirements mismatch due to increased number of stakeholders involved in the design - - -1.10.2 Implementation Constraints - -• Evaluation and testing of software options is expected to occur prior to selection and - implementation of a production urban platform. -• Budget costs are unknown until evaluation of software options is completed. - - -1.11 Assumptions, Alignment with other Action Clusters and Policies -1.11.1 Assumptions - -The assumptions in Table 3 have been identified by the Demand Side Engagement Stream as -relevant to this Requirements Specification. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 11 - - -Table 3. Assumptions - - # ASSUMPTION - - 1 The providers of city data and services will be responsible to maintain their resources in the platform. - - 2 All city data must meet the minimum metadata requirements and use the standards adopted by the platform. - - 3 The platform shall consider open Source as an optional commercial model, with open standards as a principle - - 4 The system design and architecture should minimize fragmentation of city data in the urban platform. - - 5 To the extent possible, automation should be used for the extraction of descriptive and technical metadata. - - 6 The platform must be designed in a way it accommodates additional functionality at later stage at a fair and transparent cost. - - The platform must be a modular based architecture which relies on stable and well-defined open interfaces to ensure - 7 - interoperability between the platform, services and the applications provided by service providers. - - The platform will offer open and well-documented API’s and clear service descriptions and contracts that is offered for reuse by - 8 another party to foster open innovation in the city, which means that developers and interested individuals openly utilize the - resources provided. - - 9 Adopt open and published European and International standards where possible. - - The platform must be flexible enough to accommodate different local, National and International data protection, licensing and -10 - commercialization regulations. - -11 Platform providers will monitor emerging technologies in order to maintain and improve the architecture. - - -12 Platform providers will monitor emerging information standards, including metadata standards and data interface standards. - - -13 Platform providers will monitor new commercial models for city data exploitation - - - - -1.11.2 Alignment with Citizen’s Focus Action Cluster -This specification document is aligned with the principles defined in the Citizen Focus5 Action -Cluster of the European Innovation Partnership on Smart Cities and Communities. Citizen Focus’ is -about “working together with citizens to realize public interests at the intersection of ICT, mobility -and energy in an urban environment”. We recognize citizens as owners of and participants in -the creation and delivery of city data and digital services, and we specify requirements to deliver -new digital services that will address the societal needs of cities in a positive manner that relates to -political narratives. Societal needs and wants are considered the starting point for city data service -offering by the urban platform. The requirements in this document were elicited considering - -• Human behaviour and needs as important as technology; -• The services and data that solve social problems and drive innovation; -• The mechanisms that make data and services more accessible to users; -• The factors that influences user’s experience while interacting with services provided (e.g. - usability, feeling of security and trust); - - -5 - https://eu-smartcities.eu/node/1333 - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 12 - - -1.11.3 Policies to be developed - -The following policies have been identified by the Demand Side Engagement Stream as relevant to -this Requirements Specification. - -• Data formatting and Metadata Schemes: Urban platforms will require more expansive, robust - and useful data encoding and conversion that what is available in existing data catalogues. - Data preservation policies should be developed to allow data to be stored in formats that can - be migrated, associated with metadata and ontologies to become both humans and machines - readable and understandable, ongoing monitoring for data obsolescence, and migrating data to - systems environments as needed to ensure their continued availability. Current Metadata - schemes (e.g. open data, sensory data ontologies) should be reviewed to see if it meets - current needs for city data management. It is possible that the needs of the urban platform - designed here may require new or additional schemas. - -• Data commercialisation: The commercial exploitation of city data and their funding models - are unexplored concepts that we are committed to address. There is an urgent need to define - license agreements and fair commercial and subscription models to allow interoperable open - and proprietary data to co-exist in the platform. - -• Data publication and services deployment: Policy development will be needed regarding - ingesting data and deploying data services into the platform, including which users/machines - will be authorized to submit data for publication, the minimum requirements for data submitted - by open and proprietary data publishers, and the removal of resources and services from the - platform. - - -2. Urban Platform Value Proposition, Use Cases and -Functional Requirements -2.1 From Value Proposition to Platform Specifications -This document uses goal-oriented modelling for eliciting, elaborating, structuring, specifying, -documenting, and modifying requirements. Goals represent the objectives which the urban -platform should achieve through cooperation of actors in the intended system and in the -environment. They capture, at different levels of abstraction, the various objectives the urban -platform under design should achieve. Through goals modelling we consider how the value -proposition and intended solutions connects across the stack, how the urban platform meets city -goals, why the system and its functionality are needed, and how the stakeholders’ interests may be -addressed. - -In our specification, we present the overall goal (the value proposition) that the urban platform -should aim to achieve in order to be considered as a viable final product, and a set of sub-goals -(intended solutions) it should maintain in the long run so that the overall goal can be unceasingly -achieved. By using this approach, the low-level technical requirements can be traced back to high- -level strategic objectives of the urban platform. The formal notations used in this document are: -Achieve [“Name of Overall Goal”] and Maintain [“Name of Sub-Goal”]. The requirements of the -Urban Platform are noted for each of the sub-goals, and are presented as a series of statements -regarding the capabilities needed in to achieve the overall goal of the Urban Platform. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 13 - - -2.2 Overall Goal: “City data is exploited to its full potential” -An urban platform is an organization of people and systems, which has accepted the responsibility -to preserve city data and make it available for all the stakeholders of smart cities. Ultimately, an -urban platform is a foundation for the full exploitation of city data. Hence, the major goal an Urban -Platform must achieve is “city data is exploited to its full potential”. To achieve this high level goal, -the urban platform must maintain in the long run the five sub-goals illustrated in Figure 3. - -Each one of the defined sub-goals co-enables the achievement of the specified high-level (overall) -goal of the urban platform. The sub-goals include the ingestion of city data, metadata generation, -data management, data storage, access, preservation, and administration, provision of engaging -services in the smart cities. These sub-goals are discussed in details in the following sections. - - - - Achieve [City data is exploited How we achieve the platform overall goal? - Why do we need the sub-goals? - to its full effect] - Goal - - - - - Maintain [User s experience is - Maintain [City data is provided - enhanced by the provision of - in a harmonised way] - S ub-Goal 1 S ub-Goal 5 value-added services] - co-ena bles - - - - - Maintain [Resources are managed in Maintain [City data is offered - a safe and intelligent manner] in an accessible manner] - S ub-Goal 2 S ub-Goal 4 - - - - - Maintain [City data is orchestrated - in a market place] - S ub-Goal 3 - - - - - Figure 3. Platform High-Level Goal and its respective sub-goals. - - - -2.2.1 Urban Platform Boundary - -The use case diagram illustrated in Figure 4 identifies the boundaries between the actors (either -automated or human) and the urban platform. We have arrived at the urban platform boundary by -inspecting each business use case and determining, in conjunction with the stakeholders needs, -which part of the business use case should be implemented and what part should be done by an -outsourced product (e.g. Billing System) using the framework 4. - -This task is technology agnostic and takes into account the abilities of the users/actors, the -constraints, the goals of the urban platform. Table 4 maps out the use cases with their respective -sub-goals and actors. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 14 - - - - - Manage Services - - S ub-Goal 5 - - - Data Service - Publish City Data - Provider - S ub-Goal 1 - - - <> Deploy Data Services - rules - S ub-Goal 5 - - Authenticate in the - Manage Resources <> - Platform Platform - S ub-Goal 1 Provider Database - <> Manage Infrastructure System - S ub-Goal 2 - - Utlise Data Services - City Data <> - Publisher S ub-Goal 5 - - Store City Data Management - <> - S ub-Goal 2 - Systems Services - System - - <> - City Data - Consumer - QoS Monitoring - Transmit Data - System - Register in the Platform S ub-Goal 2 - - - - - <> - rules - - <> - rules - Platform - Provider - rules - - Commercialise Data - Consume City Data - Services City Data - S ub-Goal 3 S ub-Goal 4 - Consumer - <> - - - Commercialise - City Data Discover City Data - S ub-Goal 3 - S ub-Goal 4 - - Billing - Management - System - - - - Authenticate in the Register in the - Platform Platform - - - - - Authenticate Authenticate - Register Consumers Register Publishers - Consumers Publishers - - - - - Figure 4. Simplistic overview of the use cases identified in the early stages of the platform design. - - - - -Table 4. Use Cases Mapping with Sub-Goals and Actors - Sub-Goals Use Cases ID Specialised Use Cases Actors -HIGH-LEVEL GOAL: City data is exploited to its - - - - - Publish City Data UC1 User publishes city data via data API’s - City Data Publisher - 1. City data is Register as a publisher - UC2 User manually uploads datasets - provided in a in the Platform - harmonised way - full effect - - - - - User manages resources - Manage Resources UC3 City Data Publisher - User tracks resources usage - - Store City Data UC4 - 2. City data is - managed in a safe Transmit Data UC5 - Management Systems - and intelligent - manner - Manage Infrastructure UC6 - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 15 - - - City Data Publisher - Set commercial city data - Platform Provider - Subscribe to proprietary data City Data Consumer - Commercialise City - UC7 - Data - Manage commercial data City Data Publisher - - 3. City data is Manage data subscription City Data Consumer - orchestrated in a - Data Services Provider - market place Set commercial data services - Platform Provider - Subscribe to commercial services Data Services Consumer - Commercialise Data - UC8 - Services - Manage commercial services Data Services Provider - - Manage services subscription Data Services Consumer - UC9 - Register in the Platform City Data Consumer - - - 4. City data is Discover City Data City Data Consumer - UC10 - offered in an - User consumes city data via data - accessible manner - API’s - Consume City Data UC11 City Data Consumer - User downloads datasets - 5. User’s Data Services Provider - Deploy Data Services UC12 - experience is Platform Provider - enhanced by the - Manage Services UC13 - Data Services Provider - provision of value- - added services - Utilise Data Services UC14 City Data Consumer - - - - - 2.3 SUB-GOAL 1: City data is collected in an intelligent manner - Description: The urban platform enables the owners of city data to easily publish both historic and - data streams in the platform, as well as their associated metadata. - Rationale: This sub-goal is maintained by the services and functions to accept the publication of - city data from data providers (of both open and proprietary data) and prepare the contents for - storage and management within the urban platform. Functions include receiving data, performing - quality assurance on data, verifying data formatting and document standards, associating meta- - data information, and coordinating updates to databases and resources management. - - Achieve [City data is exploited - to its full effect] - Goal - co-ena bles - - - - - How we achieve the platform overall goal? - - - - Maintain [City data is provided in - a harmonised manner] - S ub-Goal 1 - - - - -What actions can maintain the sub-goal? - - - - Achieve [Register Publisher] Achieve [Publish City Data] Achieve [Manage Resources] - UC1 UC2 UC3 - - - - - Figure 5. Sub-Goal 1 “City data is collected in an intelligent manner” refinement. - - - Drivers: Ensure data is published in an easy and uniform way. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 16 - - -Actions: For Sub-Goal 1 to be maintained in the long-run it requires the efficient realisation of use -cases: “Publish City Data” and “Manage Resources”, as shown in Figure 5. - - -2.3.1 Use Case: Register Publisher -ID: UC1 -Refines: SUB-GOAL 1: City data is collected in an intelligent manner -Pre-condition: Data Publisher is not logged in the system -Actors: Data Publishers -Rationale: Data Publishers can register in the platform and request approval to submit city data. -They provide valid registration details (to be defined) and wait for registration confirmation. -Platform Providers may authorise or not data publishers to offer both open and proprietary city data -in the platform. Data submission agreement is a formal agreement between the Data Provider and -the Urban Platform defining the terms of the content, standards, metadata creation, and license -agreement. The Urban Platform will proactively work with Data Providers to agree on the content, -quality and format of city data. Agreements between Platform and Data Providers may be -renegotiated on a periodic or ad-hoc basis. -Refines into requirements: FREQ.1 to FREQ.5. - - Use Case Basic Stimulus and Responses - 1. The platform prompts the user for a username and password or - register new account. - 2. The user selects registration option. - 3. The platform prompts user for publisher registration information (e.g. - username, password, organisation) - 4. The user enters in their information. - UC1. Register 5. Platform verifies information and creates account. - Publisher o If non-valid information, platform shows error message and - returns to step 1. - 6. Platform provider is requested to approve the account - o Platform acknowledges registration has been successful - o If non approved, platform shows error message and returns to - step 1. - 7. End of registration - - -2.3.2 Use Case: Publish City Data -ID: UC2 -Refines: SUB-GOAL 1 - City data is collected in an intelligent manner -Pre-condition: User is authenticated in the platform -Actors: City data publisher -Rationale: The Publish City Data function provides the appropriate mechanisms to receive city data from -authorized data providers. Data may be manually uploaded or submitted via APIs. In general, data providers -with whom the Urban Platform negotiates submission agreements are the providers of proprietary city data -(those producing published material, i.e. publishers) and open data, and they can be either humans or -machines. The providers of the Urban Platform will provide data providers with specifications on the content, -quality and format of data, and publication terms and conditions. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 17 - - -The Publish City Data function may represent a legal transfer of custody for the data in the urban platform, -and may require that special access controls be placed on the contents. This function provides a -confirmation of receipt of data publication to the Producer, which may include a request to resubmit data in -the case of errors resulting from the submission. -Once data has arrived, it must undergo several reviews, including virus checking, format compliance, -metadata minimum requirement agreement, quality and anticipated content and data formatting. The -platform must include the ability to record all actions and decisions made concerning the publication of city -data. The reasons for publication failure (e.g. missing metadata information, non-valid dataset) will be -provided back to the city data publisher. In some cases, the provider can then resubmit corrected data and -metadata information, while in other instances data publication refusal criteria should prevent the publisher -from submitting the same dataset at a later time period (e.g. in cases of suspicious datasets – copyrights -violation, viruses). When data is successfully submitted (either via APIs or manual upload), it will be -processed/prepared for storage into the platform’s database. -Specialised Use Cases: The Use Case Publish City Data data is distinguished into two -specialised Use Cases: “User publishes city data via data API’s (UC2.1)” and “User manually -uploads datasets (UC2.2)”. -Subordinated Use Cases: “Store City Data (UC4)” -Refines into requirements: FREQ 6 to FREQ.26. - - - Specialised Use Cases Basic Stimulus and Responses - 1. Platform provides user with an interface for static data - publication - 2. User selects datasets to be uploaded - 3. User provides metadata associated with the data (license, - provenance, ownership, semantics) in accordance with defined - standards - UC2.1. User manually 4. User requests data publication - uploads datasets 5. Platform quickly process user’s request for data publication - 6. Platform validates data submitted - o If valid data, platform acknowledges data publication has - been successful - o If non-valid data, platform shows error message and returns - to step 1. - 7. End of data publication - 1. Platform provides user with an interface for real-time data - publication - 2. User input data API information - 3. User provides metadata associated with the data (license, - provenance, ownership, semantics) in accordance with defined - standards - UC2.2. User publishes 4. User confirm information and request data publication - city data via data API’s 5. Platform quickly process user’s request for data publication - 6. Platform validates data submitted - o If valid data, platform acknowledges data publication has - been successful - o If non-valid data, platform shows error message and returns - to step 1 - End of data publication - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 18 - - -2.3.3 Use Case: Manage Resources -ID: UC3 -Refines: GOAL 1: City data is collected in an intelligent manner -Pre-condition: User successfully authenticates in the platform -Actors: City data publisher -Rationale: Manage resources provides the services and functions for updating, maintaining and -accessing both data and metadata, as well as tracking the usage of resources by users. Ideally -the owners of the resources should be the only authorised user to manage resources, and other -authorised users can track the usage of the resources in the platform. The platform must provide a -database update response indicating the status of the update, avoid update errors to be -propagated in the platform, and should keep an audit trail of all actions to enable rollback. Data -usage tracking includes performing queries on the data management data to generate result sets, -and producing reports from these result sets. -Specialised Use Cases: The Use Case Manage Resources data is distinguished into two -specialised Use Cases: “User manages resources (UC3.1)” and “User tracks resources usage -(UC3.2)”. -Subordinated Use Cases: “Transmit Data (UC5)” -Refines into requirements: FREQ.27 to FREQ.25. - - - Specialised Use Cases Basic Interactions and Responses - 1. Platform provides user with an interface for resources - management (e.g. data and metadata, data usage) - 2. User chooses to edit or delete data - 3. If edit, user revise metadata associated with the data (license, - provenance, ownership, access-control, semantics); - UC3.1. User manages 4. If delete, user selects dataset(s) to be removed - resources 5. User confirms action - 6. Platform quickly process user’s request - 7. Platform confirms execution of request - o If valid request, platform acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 8. End of resources management - 1. Platform provides user with an interface for resources - management (e.g. data and metadata, data usage) - 2. User chooses to visualise usage information of a dataset - UC3.2. User tracks 3. Platform quickly process user’s request for data usage - resources usage information - 4. Platform provides user with statistical information about data - usage and data users anonymised information - 5. End of data usage tracking. - - -2.3.4 Functional Requirements - - Req. ID UC. ID Description Priority Domain - Societal Needs, - FREQ.1 UC1 Allow data publishers to register to submit data for publication Must - Platform - Tracks data publication agreements between Data and Platform Business Needs, - FREQ.2 UC1 Must - Providers Platform - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 19 - - - Store terms of agreements, and use them to monitor/review/process - FREQ.3 UC1 Must City Data, Platform - data submissions. - Able to add and edit terms of agreement, based on access of level of Business Needs, - FREQ.4 UC1 Must - user. Platform - - FREQ.5 UC1 Data publications are managed and monitored Must City Data, Platform - - Allow authenticated users from across different organisations to City Data, Platform, - FREQ.6 UC2 Must - publish city data Business Needs - Provide authorization mechanisms for users and sensors to publish - FREQ.7 UC2 Must City Data, Platform - city data - City Data, Platform, - FREQ.9 UC2 Provide mechanisms for static data publication Must - Business Needs - City Data, Platform, - FREQ.10 UC2 Provide mechanisms for real-time data publication Must - Business Needs - - FREQ.11 UC2 Enable the publication of metadata Must City Data, Platform - - - FREQ.12 UC2 Maintain temporal information about the data Must City Data, Platform - - - FREQ.13 UC2 Support sensory data collection Must City Data, Platform - - - FREQ.14 UC2 Accepts content in numerous file types/formats Must City Data, Platform - - Prompts a request for resubmission to the data provider if an error of - FREQ.15 UC2 Must City Data, Platform - data transmission or receipt occurs - - FREQ.16 UC2 Enable the semantic description of connected devices Must City Data, Platform - - - FREQ.17 UC2 Gather data from authenticated and authorized devices Must City Data, Platform - - - FREQ.18 UC2 Validates automatically the successful transfer of the data Must City Data, Platform - - - FREQ.19 UC2 Performs virus checking on data Must City Data, Platform - - Verifies the validity of the submission based on submitter, expected - FREQ.20 UC2 Must City Data, Platform - format, data quality, and completeness - Platform should have built-in checks on the incoming metadata. Data - FREQ.21 UC2 not containing the minimally defined set of attributes should be Must City Data, Platform - returned to the publisher for metadata enhancement. - System should have a user-friendly method of mapping non-standard - FREQ.22 UC2 Should City Data, Platform - metadata elements into approved standard elements. - Once ingested, metadata should be stored in a single common format. - FREQ.23 UC2 This format should be one that ensures against data loss, and allows a Must City Data, Platform - variety of access/distribution options - Data in the repository shall have sufficient technical metadata to - FREQ.24 UC2 assure functionality (e.g. viewing and display) to ensure accessibility Must City Data, Platform - and reusability. - Allows publishers to display and perform manual/visual quality control Business Needs, City - FREQ.25 UC2 Could - assurance via a user-friendly GUI Data, Platform - Business Needs, City - FREQ.26 UC2 Any errors shall prompt a request for resubmission of data Should - Data, Platform - - FREQ.27 UC3 Enable data providers to manage their resources Must Business Needs - - A minimal set of identifying information/metadata concerning data Business Needs, - FREQ.28 UC3 Must - publication submission must be recorded Platform - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 20 - - - Stores and tracks versions of data. Links /connections between - FREQ.29 UC3 Must City Data, Platform - versions are created and maintained - Give service and data providers access to anonymized data of the Business Needs - FREQ.30 UC3 Should - subscribers of their data or services - City Data, Platform, - FREQ.31 UC3 Enable data providers to maintain and repair data and metadata Should - Business Needs - Tracks data publication agreements between Data and Platform Business Needs, - FREQ.32 UC3 Must - Providers Platform - Store terms of agreements, and use them to monitor/review/process - FREQ.33 UC3 Must City Data, Platform - data submissions. - Able to add and edit terms of agreement, based on access of level of Business Needs, - FREQ.34 UC3 Must - user. Platform - - FREQ.35 UC3 Submission volumes and schedules are managed and monitored Must City Data, Platform - - - - -2.4 SUB-GOAL 2: City data is managed in a safe and intelligent manner -Rationale: The urban platform enables users to publish, consume and commercialise data, as well -as deploy and manage services all in a secure and privacy protected manner. - - Achieve [City data is exploited - to its full effect] - Goal - co-ena bles - - - - - Maintain [Resources are managed in a - safe and intelligent manner] - S ub-Goal 1 - - - - - Achieve [Store Data] Achieve [Transmit Data] Achieve [Manage Infrastructure] - UC4 UC5 UC6 - - - Figure 6. Sub-Goal 2 “City data is managed in a safe and intelligent manner” refinement. - - - -Drivers: Ensure data is secured and the identity of users are preserved -Actions: For Sub-Goal 2 to be maintained in the long-run it requires the efficient realisation of use -cases: “Store City Data” and “Retrieve and Transmit City Data”, as shown in Figure 6. - - -2.4.1 Use Case: Store City Data -ID: UC4 -Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner -Pre-condition: Data is successfully published in the platform -Actors: Urban Platform -Rationale: When data is successfully submitted (either via APIs or manual upload), it is -processed/prepared for storage into the platform’s database. This procedure will include the - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 21 - - -generation of unique identifiers to the database, enrichment with ontologies (when applicable), -encrypted (when applicable), signed with digital certificates (when applicable) to ensure that the -data conforms to the platform data formatting, standards, security and regulation. Data may be -converted to accepted formats, as needed (e.g. graph model). A primary goal of the conversion of -content for the platform is the preservation of the content. Priority will be given to preserving the -data accordingly to the policies defined in section (2.6.2). Access-control levels and license models -are associated to data which is subject to restrictions relating to access and conditions of use. -Refines into requirements: FREQ 28 – FREQ 39. - - Use Case Basic Interactions and Responses - 1. Platform mechanisms converts submitted data into a standard format - 2. Security enforcement (e.g. anonymisation, cryptography) is placed on - sensitive information. - UC4. Store City 3. Platform associates with datasets the access-control definitions set by - Data owner of resources. - 4. Data is enriched with semantics and is associated with other datasets - 5. Platform stores data in a scalable and secure database. - 6. End of data storage. - - -2.4.2 Use Case: Transmit Data -ID: UC5 -Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner -Pre-condition: Data is successfully published in the platform -Actors: Urban Platform -Rationale: The platform accepts data retrieval request, validates user’s rights to access the data, -retrieves city data from data storage, and moves a copy of the data to the relevant platform -component for further processing. If special processing is required, the retrieval function accesses -data in staging storage and applies the requested processes. The types of operations, which may -be carried out, include sub-sampling in temporal or spatial dimensions, conversions between -different data types or output formats, and other specialized processing (e.g., data visualisation). -Once it is finalised data will be sent to the appropriate delivery channels (e.g. API’s, GUI). It also -encompasses function to verify corruption during any internal data transfer. This function requires -that all hardware and software within the platform provide notification of potential errors and that -these errors are routed to standard error logs that are checked by the Platform Provider. -Refines into requirements: FREQ 40 to 54. -2.4.3 Use Case: Manage Infrastructure -ID: UC6 -Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner -Pre-condition: The platform is available -Actors: Urban Platform -Rationale: Manage infrastructure provides the services and functions for the overall operation of -the urban platform. Administration functions include monitoring quality of service agreements, -auditing data publication to ensure that they meet archive standards, and maintaining configuration -management of system hardware and software. In overall, it provides system engineering -functions to monitor and improve platform operations, and to inventory, report on, and -migrate/update the contents of the platforms’ databases. -Refines into requirements: FREQ 55 to 62. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 22 - - - - Use Case Basic Interactions and Responses - 1. Platform keeps monitoring services at run-time to ensure operation - and integrity of city data - o If system failure, the platform activates mechanisms for recovery - UC6. Manage based on pre-defined rules - Infrastructure o Platform logs issue and issue alert messages to platform providers - 2. Platform logs operation capabilities (e.g. performance, mean of time - failure, issues, etc.) - - -2.4.4 Functional Requirements - - Req. ID UC. ID Description Priority Domain - A minimal set of identifying information/metadata concerning data Business Needs, - FREQ.28 UC4 Must - publication submission must be recorded. Platform - Stores and tracks versions of data. Links /connections between - FREQ.29 UC4 Must City Data, Platform - versions are created and maintained. - - FREQ.30 UC4 Converts data to accepted file formats Must City Data, Platform - - Keep sensitive information secured and accessible only to authorized - FREQ.31 UC4 Must City Data, Platform - users - - FREQ.32 UC4 Keep user’s personal information protected Should City Data, Platform - - - FREQ.33 UC4 Keep city data and meta-data secured Must Platform Needs - - - FREQ.34 UC4 Enable privacy preserving mechanisms associated to data Must Platform Needs - - - FREQ.35 UC4 Model data in accordance with defined standards Must City Data, Platform - - - FREQ.36 UC4 Support the use of ontologies and semantic modelling of city data Could City Data - - - FREQ.37 UC4 Support database-level provenance annotation Should City Data - - - FREQ.38 UC4 Support data-level provenance annotation Should City Data - - - FREQ.39 UC4 Enable data to be encrypted Should Platform Needs - - System must have the ability to search and display metadata, preferably - City Data, Platform - FREQ.40 UC5 in a user-conformable, human readable display as well as in its native Must - Needs - format for machine harvesting and manipulation. - Controls access to data in the repository based on multiple permission - City Data, Platform - FREQ.41 UC5 levels. These permission levels determine the create/edit/read/delete Should - Needs - privileges granted users. - Access rights and conditions of use will be held for each data and its City Data, Platform - FREQ.42 UC5 Should - related metadata. Needs - Access rights and conditions can be inherited from a parent data to any City Data, Platform - FREQ.43 UC5 Could - data designated as a child data (derived information). Needs - Access rights and conditions of use will be machine readable and City Data, Platform - FREQ.44 UC5 Should - actionable. Needs - Access mechanisms must be sufficiently granular to allow the Business Needs, City - FREQ.45 UC5 identification of individual users, in order to maintain audit logs of actions Should - Data, Platform - performed by users. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 23 - - - Maintains the integrity of the database which contains both metadata - FREQ.46 UC5 and system information. Must Platform Needs - - Provides internal validation such as referential integrity of the contents of City Data, Platform - FREQ.47 UC5 the database. Must - Needs - Creates and maintains schema definitions required to support data - FREQ.48 UC5 management functions. Must Platform Needs - - Monitors and ensures that data and metadata are not corrupted during - FREQ.49 UC5 transfers. Must Platform Needs - - Provides statistically acceptable assurance that no components of the - FREQ.50 UC5 data are corrupted during any internal data transfer. Must Platform Needs - - Performs routine and special data integrity checking for each dataset - FREQ.51 UC5 and generates error reports. Must Platform Needs - - Provides disaster recovery capabilities including data backup, off-site - FREQ.52 UC5 data storage, data recovery, etc. Must Platform Needs - - Refresh/replace data without service interruption, and update - FREQ.53 UC5 corresponding metadata as appropriate. Must Platform Needs - - Ensure that any associated unique identifiers of the updated data are not - FREQ.54 UC5 altered. Must Platform Needs - - Audits submissions to ensure that they meet archive/repository - FREQ.55 UC6 standards. Must Platform Needs - - Maintains configuration management of the system hardware and - FREQ.56 UC6 software. Must Platform Needs - - Has capability to inventory, report on and migrate the contents of the - FREQ.57 UC6 repository. Must Platform Needs - - - FREQ.58 UC6 Ensures data integrity for version upgrades and format migration. Must Platform Needs - - - FREQ.59 UC6 Monitors functionality of the entire repository. Must Platform Needs - - - FREQ.60 UC6 Maintains integrity of system configuration. Must Platform Needs - - Audits system operations, performance and usage. - FREQ.61 UC6 Must Platform Needs - - Provides platform performance information and database holdings - FREQ.62 UC6 inventory reports Must Platform Needs - - - - -2.5 SUB-GOAL 3: City data is orchestrated in a marketplace -Rationale: The urban platform enables users to consume and publish data in a secure and privacy -protected manner. -Drivers: Ensure data is secured and the identity of users are preserved -Actions: For Sub-Goal 3 to be maintained in the long-run it requires the efficient realisation of use -cases: “Commercialise City Data” and “Commercialise Data Services”, as shown in Figure 7. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 24 - - - - Achieve [City data is exploited - to its full effect] - Goal - - - - - co-ena bles - Maintain [City data is orchestrated in - a marketplace] - S ub-Goal 3 - - - - - Achieve [Commercialise Data - Achieve [Commercialise City Data] - Services] - UC7 UC8 - - - - - Figure 7. Sub-Goal 3 “City data is orchestrated in a marketplace” refinement. - - - -2.5.1 Use Case: Commercialise City Data -ID: UC7 -Refines: SUB-GOAL 3: City data is orchestrated in a marketplace -Pre-condition: Data is successfully published in the platform, both publisher and consumers of -city data are authenticated in the platform, and there are billing capabilities available. -Actors: Data Publishers, Data Consumers -Rationale: The providers of city data can commercialise city data based on the policies and -financial models defined in the platform. After publishing their data, publishers can define which -data can be available as open data and which data should be available with the payment of a -subscription fee. Once publishers define which data is to be commercially exploited, the platform -will associate the data with their respective financial models and let it ready for subscription. City -data consumer chooses which data to purchase and is redirected to a billing interface where the -subscription payment is taken. The platform must provide an update response indicating the status -of the payment. If successful, data is ready available to be consumed by humans and machines, -otherwise the user can re-try the payment or cancel transaction. -Commercialise city data also involves the function of managing commercial data. It provides -services and functions for updating, maintaining and accessing both data and its respective -commercial transactions. Furthermore, it enables data providers to track the usage of commercial -data by users. Ideally the owners of the data should be the only authorised user to manage -resources, and other authorised users can track the usage of the data in the platform. Data usage -tracking includes performing queries on the data management data to generate result sets, and -producing reports from these result sets. Data consumers are provided with functions which enable -them to manage their subscriptions and financial transactions on the platform. These functions -include updating, maintaining and accessing financial transactions. For all these functions and -services, the platform must provide a database update response indicating the status of the -update, avoid update errors to be propagated in the platform, and should keep an audit trail of all -actions to enable rollback. -Specialised Use Cases: The Use Case Commercialise City Data is distinguished into four -specialised Use Cases: “Commercialise Data (UC7.1)”, “Consume Commercial City Data (UC7.2)”, -“Manage Commercial Data (UC7.3)” and “Manage Data Subscription (UC7.4)”. -Refines into requirements: FREQ 63 to 68. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 25 - - - - - Use Case Basic Interactions and Responses - 1. User selects the data to be commercialised - 2. User selects the commercial model for data consumption - UC7.1. 3. Platform validates selection - Commercialise 4. Platform associates data to subscription model - Data 5. Platform releases data for commercial exploitation in the marketplace - 6. End of data commercialisation set up. - 1. User selects the data to be subscribed to - 2. User request subscription to data - UC7.2. 3. Platform validates selection - Consume 4. Platform redirects user to billing system - Proprietary 5. Billing system deals with user request - Data o If successful, user is redirected to a GUI where data is ready to use - o If unsuccessful, user can try payment again or cancel request - 2. End of data subscription. - 1. Platform provides user with an interface for commercial data - management - 2. User chooses to edit or delete commercial data - o If edit, user revise commercial models, licenses, access-control, - semantics; - o If delete, user selects dataset(s) to be removed (following policies - UC7.3. Manage defined in the platform for data removal) - commercial 3. User confirms action - Data 4. Platform promptly process user’s request - 5. Platform confirms execution of request - o If valid request, platform acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 6. End of resources management - 1. Platform provides user with an interface for data subscription - management - 2. User chooses to edit or cancel data subscription - o If edit, user revise payment and subscription timeframe; - o If cancel, user selects dataset(s) to have subscription cancelled - UC7.4. Manage (following policies defined in the platform for data subscription) - data 3. User confirms action - subscription 4. Platform promptly process user’s request - 5. Platform confirms execution of request - o If valid request, platform updates acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 6. End of data subscription management - - - -2.5.2 Use Case: Commercialise Data Services -ID: UC8 -Refines: SUB-GOAL 3: City data is orchestrated in a marketplace -Pre-condition: Data is successfully published in the platform, both publisher and consumers of -city data are authenticated in the platform, and there are billing capabilities available. -Actors: Data Service Providers, Data Publishers and Consumers - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 26 - - -Rationale: After deploying data services in the platform, the providers of data services can choose -to commercialise services based on the policies and financial models defined in the platform. Once -data service providers define which service(s) is (are) to be commercially exploited, the platform -will associate the services with their respective financial models and let available in the platform -applications module ready for use. User (either publisher or consumer) chooses which data -services to use, and in case a charged service is selected the platform redirects the user to a -billing interface where the payment is taken. The platform must provide an update response -indicating the status of the payment. If successful, service is ready available to be used, otherwise -the user can re-try the payment or cancel transaction. Note that data service owners should be -able to waive the payment of tariff to certain users’ categories. -Data services providers are offered with functions to manage their commercial services. The -platform provides functions for updating, maintaining and accessing both service and its respective -commercial transactions. Furthermore, it enables data services providers to track the usage of -services by users. Services usage tracking includes performing queries on the platform to -generate result sets, and producing reports from these result sets. The consumers of data services -are provided with functions which enable them to manage their subscriptions and financial -transactions. These functions include updating, maintaining and accessing financial transactions. -For all these functions and services, the platform must provide a database update response -indicating the status of the update, avoid update errors to be propagated in the platform, and -should keep an audit trail of all actions to enable rollback. -Specialised Use Cases: The Use Case Commercialise Data Services is distinguished into four -specialised Use Cases: “Commercialise Data Services (UC8.1)”, “Consume Data Services -(UC8.2)”, “Manage Commercial Services (UC8.3)” and “Manage Services Subscription (UC8.4)”. -Refines into requirements: FREQ 68 to 73. - - Use Case Basic Interactions and Responses - 1. User selects the data services to be commercialised - 2. User selects the commercial model for data service usage based on the - category of users - UC8.1. 3. Platform validates selection - Commercialise 4. Platform associates data services to subscription model - Data Services 5. Platform enables data service to be commercially exploited in the - marketplace - 6. End of services commercialisation set up. - 1. User selects the data service to be subscribed to - 2. User request subscription to service - 3. Platform validates selection - UC8.2. 4. Platform validates user’s category its respective commercial models - Consume Data 4. If applicable, platform redirects user to billing system - Services 5. Billing system deals with user’s request - o If successful, user is redirected to a GUI where data is ready to use - o If unsuccessful, user can try payment again or cancel request - 6. End of data subscription. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 27 - - - 1. Platform provides user with an interface for commercial data - management - 2. User chooses to edit or delete commercial data - o If edit, user revise commercial models, licenses, access-control, - semantics; - o If delete, user selects dataset(s) to be removed (following policies - UC8.3. Manage defined in the platform for data removal) - commercial 3. User confirms action - services 4. Platform promptly process user’s request - 5. Platform confirms execution of request - o If valid request, platform acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 6. End of resources management - 1. Platform provides user with an interface for data subscription - management - 2. User chooses to edit or cancel data subscription - o If edit, user revise payment and subscription timeframe; - o If cancel, user selects dataset(s) to have subscription cancelled - UC8.4. Manage (following policies defined in the platform for data subscription) - services 3. User confirms action - subscription 4. Platform promptly process user’s request - 5. Platform confirms execution of request - o If valid request, platform updates acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 6. End of data subscription management - - -2.5.3 Functional Requirements - - Req. ID UC. ID Description Priority Domain - City Data, Platform, - FREQ.63 UC7 Support the commercialization of city data Should Business Needs - City Data, Platform, - FREQ.64 UC7 Enable users to subscribe to city data through the payment of a tariff Should Business Needs - City Data, Platform, - FREQ.65 UC7 Enable users to manage their data subscriptions Should Business - Provide platform providers mechanisms to define the terms and - FREQ.66 UC7 Must Platform Needs - conditions for platform data usage - Enable data providers to manage the subscription models of their City Data, Platform, - FREQ.67 UC7 Should - data Business - UC7/ City Data, Platform, - FREQ.68 Utilise secure and reliable billing and payment management systems Must - UC8 Business - City Data, Platform, - FREQ.69 UC8 Support the commercialization of data services Should Business Needs - Enable data providers to manage the commercial models of their City Data, Platform, - FREQ.70 UC8 Should - services Business Needs - Provide service providers mechanisms to define the terms and City Data, Platform, - FREQ.71 UC8 Should - conditions of platform services Business Needs - Allow users to pay a tariff for using certain advanced services (e.g. City Data, Platform, - FREQ.72 UC8 Should - Data manipulation, enrichment) Business Needs - City Data, Platform, - FREQ.73 UC8 Enable users to manage their data services subscriptions Should Business Needs - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 28 - - -2.6 SUB-GOAL 4: City data is offered in an accessible manner -Rationale: The urban platform provides city data in both human and machine (e.g. sensors, -actuators, systems) readable and understandable formats. -Drivers: Ensure data understandability and machine-to-machine data transaction. -Actions: For Sub-Goal 3 to be maintained in the long-run it requires the efficient realisation of use -cases: “Register Consumer”, “Discover City Data”, and “Consume City Data” as shown in Figure 8. - - Achieve [City data is exploited - to its full effect] - Goal - - - - - co-ena bles - Maintain [City data is offered in an - accessible manner] - S ub-Goal 1 - - - - - Achieve [Register Consumer] Achieve [Discover City Data] Achieve [Consume City Data] - UC9 UC10 UC11 - - - - - Figure 8. Sub-Goal 4 “City data is offered in an accessible manner” refinement. - - - -2.6.1 Use Case: Register Data Consumer -ID: UC9 -Refines: SUB-GOAL 4: City data is offered in an accessible manner -Pre-condition: User is not logged in the platform -Actors: Data Consumers -Rationale: Data Consumers can register in the platform and request approval to consume city data -via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to -confirm their registration. Users must accept the terms and conditions of platform usage and define -how their personal data can be used by the Platform Owner. Users can manage and alter their -registration information at any time they want to. -Refines into requirements: FREQ 64. - - Use Case Basic Stimulus and Responses - 1. The platform prompts the user for a username and password or register - new account. - 2. The user selects registration option. - 3. The platform prompts user for data consumer registration information - UC1. Register 4. The user enters in their information. - Consumer 5. Platform verifies information and creates account. - o If non-valid information, platform shows error message and returns to - step 1. - 6. Platform acknowledges registration has been successful - 7. End of registration - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 29 - - -2.6.2 Use Case: Discover City Data -ID: UC10 -Refines: SUB-GOAL 4: City data is offered in an accessible manner -Pre-condition: User has access to either platform GUI or API -Actors: Data Consumers -Rationale: Data Consumers can register in the platform and request approval to consume city data -via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to -confirm their registration. -Refines into requirements: FREQ 64. - - Use Case Basic Stimulus and Responses - 1. Users access specialised data query end-points (e.g. SPARQL) - 2. Users provides information for pre-defined parameters for search - 3. Users request data search - 4. Platform quickly process users request for data - UC10.1 o All queries are verified against access rights restrictions - Discover city o If restriction applies users are redirected to log in interfaces - data via data o Users provide credentials and log on the system - query end-points 5. Users are provided with query results on the end-point if access is - allowed - o If access is not not allowed platform issues an error message to the - user - 1. Users search city data via GUI - 2. Users inputs search parameters (e.g. key words, categories, formats, - publishers) - UC10.2 3. Users request data search - Discover city 4. Platform quickly process users request for data - data via data o All queries are verified against access rights restrictions - query end-points o If restriction applies users are redirected to log in interfaces - o Users provide credentials and log on the system - 5. Users are provided with query results on an interface - o If access is not allowed platform issues an error message to the - user - - - -2.6.3 Use Case: Consume City Data -ID: UC11 -Refines: SUB-GOAL 4: City data is offered in an accessible manner -Pre-condition: User has access to either platform GUI or API -Actors: Data Consumers -Rationale: Data Consumers can register in the platform and request approval to consume city data -via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to -confirm their registration. -Refines into requirements: FREQ 64. -Subordinated Use Cases: “Consume proprietary city data” - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 30 - - - - Use Case Basic Stimulus and Responses - 1. Users request data to be formatted in a particular format supported by - UC11.1 the platform - Customise City 2. Platform quickly process users request for data formatting - Data 3. Mechanism for data conversion is called and process data - 4. Users are provided with data formatted as requested - 1. Users / Machines select data to be downloaded - 2. Users / Machines are redirected to authentication mechanism in case of - UC11.2 registration is needed for the particular dataset - Consume City o If authentication is successful, users are provided with requested - Data via GUI data streams - 3. Users are provided with requested data via APIs - 1. Users / Machines makes data request on the platforms API - 2. Users / Machines are redirected to authentication mechanism in case of - UC11.3 registration is needed for the particular dataset - Consume City o If authentication is successful, users are provided with requested - Data via APIs data streams - 3. Users are provided with requested data via APIs - - - -2.6.4 Functional Requirements - - Req. ID UC. ID Description Priority Domain - Allow users to register to use services and consume proprietary city Societal Needs, - FREQ.74 UC9 Should - data and open data (optional) Platform - Keep sensitive information secured and accessible only to Societal Needs, - FREQ.75 UC9 Should - authorized users Platform - Societal Needs, - FREQ.76 UC9 Provide authentication mechanisms for users Must Platform - Societal Needs, - FREQ.77 UC9 Keep user’s personal information protected Must Platform - Allow users to control which data they are willing to provide and how Societal Needs, - FREQ.78 UC9 Must - their data should be used Platform - Societal Needs, - FREQ.79 UC10 Allow users to format data in any supported data formats Must - Platform - The query request may require data to be sourced from different Societal Needs, - FREQ.80 UC10 Must - storage locations Platform - Allows query requests against all metadata used to manage the Societal Needs, - FREQ.81 UC10 repository. Should Platform - Provide users information about the legal aspects of the data Societal Needs, - FREQ.82 UC11 Must - (license, ownership) City Data - Societal Needs, - FREQ.83 UC11 Keeps an audit trail of all actions. Must City Data - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 31 - - -2.7 SUB-GOAL 5: User’s experience is enhanced by the provision of -value-added services -Rationale: The urban platform enables users to consume and publish data in a secure and privacy -protected manner -Drivers: Ensure data is secured and the identity of users are preserved -Actions: For Sub-Goal 5 to be maintained in the long-run it requires the efficient realisation of use -cases: “Deploy Data Services”, “Manage Services”, and “Utilise Data Services” as shown in Figure -8. - - Achieve [City data is exploited - to its full effect] - Goal - - - - - co-ena bles - Maintain [User s experience is enhanced by - the provision of value-added services] - S ub-Goal 1 - - - - - Achieve [Deploy Data Services] Achieve [Manage Services] Achieve [Utilise Data Services] - UC12 UC13 UC14 - - - - - Figure 9. Sub-Goal 5 “User’s experience is enhanced by the provision of value-added services” refinement - - - -2.7.1 Deploy Data Services -ID: UC12 -Refines: SUB-GOAL 5: User’s experience is enhanced by the provision of value-added services -Pre-condition: Data Services Providers are provided with credentials, are authorised to deploy -their services in the platform, and have access to technical specifications of the platform interfaces. -Actors: Data Services Providers -Rationale: Data Services can register in the platform and request approval to deploy services in -the platform. They provide valid registration details (to be defined) and wait for registration -confirmation. Platform Providers may authorise or not data services providers to offer both open -and proprietary services in the platform. Data services providers must formally agree with service -deployment agreement with the Urban Platform. This agreement defines terms of the content, -policies, regulations, license agreement. The Urban Platform will proactively work with Service -Providers to agree on the technical specifications of interfaces and platform openness level. -Agreements between Platform and Data Service Providers may be renegotiated on a periodic or -ad-hoc basis. -Refines into requirements: FREQ.1 to FREQ.5. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 32 - - - - Use Case Basic Stimulus and Responses - 1. User authenticates in the platform - 2. User is provided access to platform interfaces for service deployment - 3. User deploy services - 4. Platform checks compatibility and any technical issues arising from the - UC12. Deploy new service - Services 5. If approved the service is ready to be used and managed in the - platform - o If deployment is not approved, platform shows error message and - returns to step 1. - 6. End of deployment - - - -2.7.2 Use Case: Manage Services -ID: UC13 -Refines: GOAL 5: User’s experience is enhanced by the provision of value-added services -Pre-condition: User successfully authenticates in the platform -Actors: Data Services Providers -Rationale: Manage services provides the functions for updating, maintaining and accessing -services as well as tracking their usage by users. Ideally the owners of the services should be the -only authorised user to manage them, and other authorised users can track the usage of the -services in the platform. In case of updates the platform must log in the database update details -and send to service providers a response indicating the status of the update. The platform should -also ensure update errors are not propagated nor affect the health of other services provided in the -platform, and should keep an audit trail of all actions to enable rollback. Data services usage -tracking includes performing queries on the data management data to generate result sets, and -producing reports from these result sets. All user’s information provided to service providers must -follow regulations of data protection and the user’s defined rules for their data use. -Specialised Use Cases: The Use Case Manage Services is distinguished into two specialised -Use Cases: “User manages services (UC13.1)” and “User tracks service usage (UC13.2)”. -Subordinated Use Cases: “Transmit Data (UC5)” -Refines into requirements: FREQ.27 to FREQ.25. - - - Specialised Use Cases Basic Interactions and Responses - 1. Platform provides user with an interface for services management - 2. User chooses to edit or delete services - 3. If edit, user revise service information (access-control, - commercial models, parameters) and deployment; - If delete, user selects services to be removed / disabled - UC13.1. User manages 4. User confirms action - services 5. Platform quickly process user’s request - 6. Platform confirms execution of request - o If valid request, platform acknowledges request has been - processed successfully - o If non-valid request, platform returns to step 1. - 7. End of services management - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 33 - - - 1. Platform provides user with an interface for services management - 2. User chooses to visualise usage information of a service - 3. Platform quickly process user’s request for data usage - UC13.2. User tracks information - services usage 4. Platform provides user with statistical information about services - usage and data users anonymised information - 5. End of data services tracking. - - - -2.7.3 Use Case: Utilise Data Services -ID: UC2 -Refines: SUB-GOAL 1 - City data is collected in an intelligent manner -Pre-condition: User is authenticated in the platform -Actors: City data publisher -Rationale: Data Consumers can register in the platform and request approval to consume city data via GUI -or APIs. They provide valid registration details (to be defined) and wait for platform to confirm their -registration. -Subordinated Use Cases: “Commercialise Data Services (UC8)” -Refines into requirements: FREQ 6 to FREQ.26. - - - Specialised Use Cases Basic Interactions and Responses - 1. Users / Machines select data service to be utilised - 2. Users / Machines are redirected to authentication mechanism in - UC2.1. User utilises data case of registration is needed for the particular service - services o If authentication is successful, users are provided with - requested data streams - 3. Users are provided with requested service either via API or GUI - - -2.7.4 Functional Requirements - - Req. ID UC. ID Description Priority Domain - Provide stable and well-defined interfaces to ensure interoperability - Societal Needs, - FREQ.84 UC12 between the platform, services and the applications provided by Should Platform - services providers - Ensure the interfaces of the architecture are open to reduce entry Societal Needs, - FREQ.85 UC12 Should - barriers and integration issues Platform - Provide multi-purposed and network intelligent interfaces to Societal Needs, - FREQ.86 UC12 Must - providers and consumers of services Platform - Provide service providers mechanisms to define the terms and Societal Needs, - FREQ.87 UC12 Must - conditions of platform services deployment Platform - Business Needs, - FREQ.88 UC13 Provide statistical information of user’s feedback on service usage Must Platform - Allow users to use services to manipulate city data (e.g. Create Societal Needs, - FREQ.89 UC10 Must - mash ups, integrate) Platform - Allow users to provide feedback on usability, and quality of data Societal Needs, - FREQ.90 UC14 Must - and services provided by the platform Platform - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 34 - - - -3. Non-functional Requirements -3.1 Run-time Quality Requirements -3.1.1 Scalability Requirements - -Rationale: The ability of the system to execute its task within its expected performance profile and -to handle on-demand increased processing volumes of data and service requests -Drivers: Provide ready access to all data that underpins decision making processes in smart cities, -and accommodate users and data usage patterns -Refines into requirements: NFREQ.1, NFREQ.2, NFREQ.3, NFREQ.4 -Relevance: Urban platforms have ambitious performance requirements. Such platforms must cope -with user’s demand for data and services, capture real-time data that will be catalysed by a myriad -of sensors. The demand for urban platform is very likely to significantly increase over time. It is -very difficult to have clear performance characteristics due to the ubiquity, heterogeneity high -connectivity of devices and end users. - - SCALABILITY MEASURES - Actions Capture the performance requirements - Create service level agreements - Predict scalability using software simulation - Analyse the performance of the platform overtime - Conduct practical testing - Strategy Prioritize service and data requests - Distribute processing over time - Scale up or scale out as necessary - Reuse resources and results Partition and parallelize - Constantly monitor Quality of Service at runtime - - - -3.1.2 Availability and Reliability Requirements - -Rationale: The ability of the system to be fully or partly operational as and when required and to -effectively handle failures that could affect system availability -Drivers: Build a reliable foundation for “on demand” exploitation of data -Refines into requirements: NFREQ.5, NFREQ.6, NFREQ.7, NFREQ.8 -Relevance: Any system that has complex or extended availability requirements, complex recovery -processes, or a high profile (e.g., is visible to the public) - - AVAILABILITY AND RELIABILITY MEASURES - Actions Capture the availability requirements Produce the availability schedule - Estimate platform availability Estimate functional availability Assess against the - requirements Rework the architecture - Strategy Adopt fault-tolerant hardware - Use reliable infrastructure database - Log transactions - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 35 - - - Develop adaptive software to cope and recover from faults - Design and test for failure - Deploy load balancing - Identify suitable backup and disaster recovery solution - - -3.1.3 Trust Requirements - -Rationale: A quality related to the user’s belief in the reliability, integrity and ability of the functional -behaviour of the platform -Drivers: Gain understanding of what influences user’s experience while interacting with services -provided -Requirements: NFREQ.16, NFREQ.17 -Relevance: Relevant to the systems that share and collect information that may raise public -concern. In some cases, trust has to do with the reliability of data and their providers, whereas in -other cases trust can be associated with the security and privacy of the technology that was -deployed. Trust affects the reputation of the platform besides its dissemination and maturity on the -market. - -TRUST MEASURES - Capture trust requirements - Perform risk analysis so measures can be implemented -Actions Explore the vulnerability aspects of city data - Check whether extensibility requirements impact on trust - Define a trust model - Adopt trust model - Deploy monitoring capabilities and assess its impact on scalability - Manage the data in a way that ensures its compliance with data -Strategy protection regulations - Implement tampering and data misuse detection - Make use of Cryptography when necessary - Allow Federation of trust between platforms - - - -3.1.4 Security Requirements - -Rationale: Ability of the system to enforce the intended confidentiality, trust, integrity and service -and data access policies, and to detect and recover from failure in these security mechanisms. -Drivers: Manage the data and services in a way that ensures its integrity, and compliance with -data protection regulations -Relevance: Relevant to the systems that share and collect information that may raise public -concern. Urban platforms may become a valuable target for attackers which can potentially leave -huge swathes of information exposed. It could potentially undermine trust in the government and -damage its reputation. -Refines into requirements: NFREQ.9 to NFREQ.15 - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 36 - - - SECURITY MEASURES - Elicit the security requirements - Cross-check security requirements’ impact on services offered by different - providers - Verify security impact on service composition - Conduct risk analysis and impacts of security breach - Use scalable and efficient user authentication components - Use authentication and authorization components to secure interfacing with - Actions external services - Address aspects of data collection, storage and distribution - Address aspects of service and communication security among devices - Identify security requirements of the physical infrastructure - Balance and prioritize scalability (performance) and security requirements - Evaluate trade-offs between privacy considerations and preventing abuse. (While - anonymous access guarantees privacy of users, traceability of users due to abuse - of the service is not possible.) - Toughen user’s functional components - Authenticate subjects - Define and enforce access policies - Secure communication infrastructures - Secure interfaces with external systems and services - Strategy - Secure databases - Check data integrity for critical services - User digital certificates and encryption where necessary - Secure monetary transactions - Secure user’s personal information - - -3.1.5 Privacy Requirements - -Rationale: Ability of the system to ensure that the collection and transmission of personal data is -minimized and handled in accordance with user’s expectation and regulations. -Drivers: Protect the vulnerability aspects volunteered citizen’s data -Requirements: NFREQ.18, NFREQ.19, NFREQ.20, NFREQ.21, NFREQ.22 -Relevance: Ensuring users privacy is protected positively influences user’s experience, -acceptance and continuous use of the platform. Besides other factors, the reputation of the -platform depends on how well user’s information is secure and preserved. - - PRIVACY MEASURES - Capture trust requirements - Perform risk analysis so measures can be implemented - Actions Explore the vulnerability aspects of city data - Check whether extensibility requirements impact on trust - Define a trust model - Allow users to interact with the platform anonymously in given - circumstances - Manage users identification securely - Strategy Anonymize data whenever necessary to comply with data regulations - Cryptograph personal identifiers during data transmission when that is - necessary - Avoid unauthorized access to implicit information (e.g. location) - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 37 - - - Verify the impact of security, trust and scalability requirements trade-offs - on privacy - Allow the user to control how personal information is used - Empower user to control the data disclosure mechanism - - - -3.2 Non Run-time Quality Requirements -3.2.1 Evolvability Requirements - -Rationale: The ability of the platform to withstand and easily adapt when new requirements and -changes is introduced. -Drivers: Ensure the platform is able to accommodate additional functionality and emerging -technologies at later stage at a fair and transparent cost -Refines into requirements: NFREQ.23 -Relevance: Important for longer- lived and more widely used systems. Urban platforms are -expected to be highly evolvable in order to accommodate future emerging technologies and avoid -interoperability issues. - - EVOLVABILITY MEASURES - Actions Characterize and assess the evolution needs - Consider the evolution trade-offs - Balance and negotiate potential conflicting requirements emerging from - the evolution capability. - Strategy Adopt standards and open interfaces - Design loose-coupled components - Preserve the platform resilience - - - -3.2.2 Extensibility Requirements - -Rationale: The flexibility of the system to allow services and functionality to be extended and -augmented by service providers in order to increase value of services to both platform providers -and end-users. The extension of the platform services is determined by the Platform Openness -strategy. -Drivers: Stakeholders can extend the services provided by the urban platform, so that -partnerships can be built to deliver holistic and interoperable solutions. Identify integrated -approaches to design and service delivery which ensures that services fit together and that -synergies can be exploited. -Refines into requirements: NFREQ.24, NFREQ.25 -Relevance: Important for longer- lived and more widely used systems. Urban platforms are -expected to be highly evolvable in order to accommodate future emerging technologies and avoid -interoperability issues. - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 38 - - - EXTENSIBILITY - Actions Characterize and assess the extensibility needs - Trace the impacts and cost of extensions - Negotiate potential conflicting requirements emerging from adding extensions - in the platform - Strategy Develop a modular architecture with standard interfaces that reduces entry - barriers due to increased transparency and integration - Share technical information about interfaces will help service providers in - targeting opportunities around the platform - - - -3.3 List of Non-Functional Requirements - -ID # Description Concern Priority Domain - - Platform, -NFREQ.1 Support different service level agreements (SLA) Scalability Should - Infrastructure - Platform, -NFREQ.2 Process services and events on a set of distributed nodes Scalability Should - Infrastructure - Platform, -NFREQ.3 Continuously monitor Quality of Service at runtime Scalability Should - Infrastructure - Platform, -NFREQ.4 Balance its load at runtime Scalability Should - Infrastructure - Platform, -NFREQ.5 Provide high availability Availability Should - Infrastructure - Platform, -NFREQ.6 Guarantee infrastructure availability Availability Should - Infrastructure - Platform, -NFREQ.7 Ensure network availability Availability Should - Infrastructure - Platform, -NFREQ.8 Be able to perform self-healing Availability Should - Infrastructure - City Data, -NFREQ.9 Expose data and services to authorized users Security Must - Platform - City Data, -NFREQ.10 Ensure services are always accessible to entitled users Security Must - Platform - -NFREQ.11 Ensure Data Freshness Security Must Platform - -NFREQ.12 Support access control mechanisms Security Must Platform - - Platform, -NFREQ.13 Have security mechanisms to protect data transmission Security Should - Infrastructure - Platform, -NFREQ.14 Make it difficult to spy on communicated messages Security Should - Infrastructure - Platform, -NFREQ.15 Be able to perform to detect threats at runtime Security Should - Infrastructure - Provide trusted and secure communication and information Platform, -NFREQ.16 Trust Should - management Infrastructure - -NFREQ.17 The platform infrastructure and services shall be trustable Trust Should Infrastructure - - Societal Needs, -NFREQ.18 Allow users to use free services anonymously Privacy Should - Platform - Societal Needs, -NFREQ.19 Allow people to use free services anonymously Privacy Should - Platform - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 39 - - - Allow users to control which data they are willing to provide Societal Needs, -NFREQ.20 Privacy Should - and how their data should be used Platform - -NFREQ.21 Keep users access-control rights/ policies secured. Privacy Should Platform - - Provide privacy protection for users interacting with the Societal Needs, -NFREQ.22 Privacy Should - platform Platform - Platform, -NFREQ.23 Provide communication confidentiality Privacy Should - Infrastructure - Societal Needs, -NFREQ.24 Be extensible for future technologies. Evolvability Must City Data, Platform, - Infrastructure - Platform, -NFREQ.26 Provide standard interfaces for service providers Extensibility Should - Business Needs - -NFREQ.26 Be able to provide services in an interoperable manner Extensibility Should Platform - - - - -4. Other Requirements -4.1 Minimal Descriptive Metadata Required from Data Provider - -Minimal Descriptive Metadata Required from Data Provider - - Attribute Required Definition Notes Examples - -Database Name Y A name given to the resource - -Author Name of a person or body associated with - Y - the creation of the resource - -Maintainer Name of the entity responsible for making - Y - the resource available - -Date Created Y Date the dataset was created - - -Date Modified Y Date the dataset was modified - - -Last Revision Y Date the dataset was last revised - - -Update Frequency Y Frequency of data maintenance - - -Mode of Release Y Open/Proprietary - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 40 - - -4.2 Urban Platform Supported Data Formats - - -MIME type Description Extensions Level - -application/json JavaScript Object Notation json supported - -application/xml eXtensible markup language file xml supported - -text/csv Comma separated values file csv supported - - - - -5. Conclusion & Forward Plans -This document represents the first set of requirements specification for urban platform. Future -activities will collaboratively assess, resolve requirements conflicts, prioritize, and validate the -requirements of the urban platform. Ultimately, this document will become a complete final -requirements specification document to guide and speed up the development open platform for -cities. Table below shows the milestones, deliverables and engagement activities completed and -yet to be completed by the Demand Side Engagement Stream. - - - - Milestones, deliverables and engagement activity Forecast date Status - Stakeholders invitation Late Completed - September/2015 - Online Workshop 1: Project Scope and Outcomes 20/11/2015 Completed - Online meeting held with city members to communicate goals, - expected outcomes, time commitment and required actions. - Start of Collaborative Requirements Engineering Process 20/11/2015 Completed - Participants reviewed the first draft of urban platform requirements, - provided comments, and suggested new requirements and changes. - Online Workshop 2: Review and Refine Requirements 08/12/2015 Completed - Participants had a conference call to review the requirements - specification document and provide their comments. - Deliverable: Requirements Specification Document v2.1 shared 05/01/2016 Completed - across the Working Streams for consultation - Online Workshop 3: Review Requirements Specification 12/01/2016 Completed - Document and discussion of Letter of Intent - Participants had a conference call to review the requirements - specification document and provide their comments, and discussed - the Letter of Intent to be signed by Mayor/Equiv of EU cities to - commit to application of common U.P. approach. - Urban Platforms Workshop in Brussels 19/01/2016 Completed - Participants provided feedback on the requirements specification - document and recommended changes in the document - Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 41 - - - Deliverable: Updated Requirements Specification Document v2.2 29/01/2016 Completed - shared with Industry and Standardisation Working Streams - Deliverable: Online release of the Requirements Specification Early - Document and open calls for wider consultation on the Requirements February/2016 - - Online Workshop 4: Engagement with Industry Side Early - Capture industry feedback on the Requirements Specification February/2016 - Document to assess its suitability to drive the development of the first - draft reference architecture. - Online Workshop 5: Review Requirements Specification Mid - Document v2.2 and discuss requirement prioritization and February/2016 - balancing - Collectively determine which candidate requirements of urban - platforms should be prioritized as high importance to all the cities - (use of Value Oriented Prioritization Method). Requirements are also - prioritized to minimize risk during development of urban platforms so - that the most important requirements are made explicit and - considered in the reference architecture. - Capture of case studies from cities intended to support requirements Mid March/2016 - validation (City of Porto), and learning and confidence building - amongst cities. - \ No newline at end of file diff --git a/go.mod b/go.mod index b42cb37..5d02ea0 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.23 require ( github.com/larksuite/oapi-sdk-go/v3 v3.5.3 + github.com/openai/openai-go v1.12.0 modernc.org/sqlite v1.34.5 ) @@ -15,7 +16,11 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/sys v0.22.0 // indirect + github.com/tidwall/gjson v1.14.4 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect + golang.org/x/sys v0.29.0 // indirect modernc.org/libc v1.55.3 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.8.0 // indirect diff --git a/go.sum b/go.sum index 0f3dd0b..866bae7 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,20 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/openai/openai-go v1.12.0 h1:NBQCnXzqOTv5wsgNC36PrFEiskGfO5wccfCWDo9S1U0= +github.com/openai/openai-go v1.12.0/go.mod h1:g461MYGXEXBVdV5SaR/5tNzNbSfwTBBefwc+LlDCK0Y= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -38,8 +50,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/internal/agent/orchestrator.go b/internal/agent/orchestrator.go index 0277043..1bccfd1 100644 --- a/internal/agent/orchestrator.go +++ b/internal/agent/orchestrator.go @@ -2,6 +2,7 @@ package agent import ( "context" + "encoding/json" "fmt" "runtime" "sort" @@ -43,14 +44,6 @@ type pendingFileRef struct { MimeType string } -type capabilityRoutingResult struct { - NeedSkills bool - SelectedToolNames []string - SelectedSkills []knowledge.Skill - Reason string - UsedFallback bool -} - type filePromptContext struct { Summary string FatalReason string @@ -110,11 +103,25 @@ func NewOrchestrator( // - 是否需要调用工具（action + action_input） // 循环持续进行，直到 LLM 返回 is_final_answer=true。 func (o *Orchestrator) HandleMessage(ctx context.Context, chatID, userID, text string) (string, error) { - return o.handleMessageInternal(ctx, chatID, userID, text, nil) + return o.handleMessageInternal(ctx, chatID, userID, text, nil, false) } func (o *Orchestrator) HandleMessageWithFiles(ctx context.Context, chatID, userID, text string, files []llm.InputFile) (string, error) { - return o.handleMessageInternal(ctx, chatID, userID, text, files) + return o.handleMessageInternal(ctx, chatID, userID, text, files, false) +} + +// HandleMessageWithFileIDs 接收用户文本与外部 file_id 列表，复用统一 ReAct 链路。 +// 该方法会先把 file_id 注入当前会话上下文，然后调用常规 HandleMessage 流程。 +func (o *Orchestrator) HandleMessageWithFileIDs(ctx context.Context, chatID, userID, text string, fileIDs []string) (string, error) { + ids := nonEmptyIDs(fileIDs) + if len(ids) > 0 { + refs := make([]pendingFileRef, 0, len(ids)) + for _, id := range ids { + refs = append(refs, pendingFileRef{ID: id}) + } + o.appendPendingFiles(chatID, userID, refs) + } + return o.handleMessageInternal(ctx, chatID, userID, text, nil, true) } // UploadAndCacheFiles 上传文件到 LLM 并缓存 file_id，供后续同会话文本问答复用。 @@ -135,7 +142,7 @@ func (o *Orchestrator) UploadAndCacheFiles(ctx context.Context, chatID, userID s return ids, nil } -func (o *Orchestrator) handleMessageInternal(ctx context.Context, chatID, userID, text string, files []llm.InputFile) (string, error) { +func (o *Orchestrator) handleMessageInternal(ctx context.Context, chatID, userID, text string, files []llm.InputFile, appendFileIDText bool) (string, error) { // 为链路追踪设置唯一的 TraceID traceID := logger.NewTraceID() ctx = logger.WithTraceID(ctx, traceID) @@ -228,9 +235,7 @@ func (o *Orchestrator) handleMessageInternal(ctx context.Context, chatID, userID } return finalText, nil } - routeInput := composeRouteInput(text, fileCtx.Summary) - route := o.routeCapabilities(ctx, routeInput) - response, err := o.runUnifiedReAct(ctx, chatID, userID, compressed, text, fileCtx, routeInput, route) + response, err := o.runUnifiedReAct(ctx, chatID, userID, compressed, text, fileCtx, appendFileIDText) if err != nil { if o.log != nil { o.log.Errorf("%s message generation failed chat_id=%s err=%v", traceLogPrefix, chatID, err) @@ -256,128 +261,198 @@ func (o *Orchestrator) handleMessageInternal(ctx context.Context, chatID, userID } // buildUnifiedSystemPrompt 构建统一 ReAct 循环的 system prompt。 -// 工具始终可用；技能仅按当前问题挑选相关项作为增强上下文。 -func (o *Orchestrator) buildUnifiedSystemPrompt(userInput string, route capabilityRoutingResult) string { +// 工具定义通过 API 的 tools 字段传递；此处只需包含人格、技能、运行环境和思考指引。 +func (o *Orchestrator) buildUnifiedSystemPrompt(userInput string) string { skillMetaDoc := o.formatSkillSummariesForPrompt() - relevantSkillsDoc := o.formatSelectedSkillsForPrompt(userInput, route.SelectedSkills) - toolDoc := o.formatToolDoc() + relevantSkillsDoc := o.formatSelectedSkillsForPrompt(userInput, nil) runtimeDoc := formatRuntimeContextForPrompt() - routeDoc := formatRouteForPrompt(route) return strings.Join([]string{ "你是一个个人自动化助手，必须遵循如下人格设定并保持一致：", o.soul, "", + "===== ReAct 思考指引 =====", + "你采用 ReAct（Reasoning + Acting）模式进行任务处理。", + "1. 思考优先：在做出任何行动之前，先在回复中阐述你的推理过程（Thought）。", + "2. 工具调用：如果需要获取信息或执行操作，使用提供的工具函数（function calling）进行调用。", + "3. 观察反馈：检查工具返回的结果，据此决定下一步行动。", + "4. 最终回答：当你有足够信息时，直接给出面向用户的最终文本回复，不要调用工具。", + "", + "注意事项：", + "- 每次要么调用工具，要么给出最终回答，不要两者都做。", + "- 如果工具调用失败，根据错误信息（Traceback）调整策略后重试或给出替代方案。", + "- 涉及文件、目录、命令时，优先调用工具获取真实结果，不要猜测。", + "- 你的思考过程（Thought）应写在回复内容中，帮助追踪推理逻辑。", + "", "===== 运行环境 =====", runtimeDoc, "", "===== 可用技能概览 =====", skillMetaDoc, "", - "===== 能力路由结果 =====", - routeDoc, - "", "===== 本轮相关技能（按用户问题筛选） =====", relevantSkillsDoc, - "", - "===== 可用工具 =====", - toolDoc, - "", - "===== 输出格式约束 =====", - "你必须使用 ReAct（Reasoning + Acting）模式进行决策。", - "每次回复必须是且仅是一个 JSON 对象，字段如下：", - "", - "{", - " \"thought\": \"你的推理过程（必填）\",", - " \"action\": \"要调用的工具名称，如 file/shell/web_search（不调工具时填 none）\",", - " \"action_input\": \"传给工具的输入（字符串或对象），不调工具时填空字符串或 null\",", - " \"is_final_answer\": true 或 false,", - " \"final_answer\": \"当 is_final_answer=true 时填写给用户的最终回复，否则填 null\"", - "}", - "", - "决策规则：", - "1) 如果你可以直接回答用户问题（不需要任何工具）：", - " 设 is_final_answer=true，action=\"none\"，final_answer 填写完整回复。", - "2) 优先判断是否可通过原子工具能力完成任务；若可完成，直接进行工具调用链路。", - "3) 当纯工具调用无法满足时，再结合已加载的技能详细说明进行决策。", - "4) 如果你需要调用工具获取信息后才能回答：", - " 设 is_final_answer=false，action 填工具名，action_input 填工具所需输入，final_answer=null。", - "5) 不要在 JSON 之外输出任何内容。", - "6) 根据技能说明中的指引决定何时以及如何使用工具。", - "7) 工具能力是全局可用的，不依赖技能命中；当技能不匹配时，仍可直接选择合适工具。", - "8) 若技能中存在与当前运行环境不匹配的章节（如 Windows 专章），应降低优先级，除非用户明确要求该环境。", - "9) 每轮工具调用结果会以 Observation 的形式追加到推理记录中，供你下一轮决策参考。", }, "\n") } -// runUnifiedReAct 执行统一的 ReAct 循环。 -// LLM 每次都看到完整的技能集+工具集，自行决定是否调用工具或直接回答。 -// 循环持续到 is_final_answer=true 或达到安全上限。 -func (o *Orchestrator) runUnifiedReAct(ctx context.Context, chatID, userID, compressedContext, userInput string, fileCtx filePromptContext, routeInput string, route capabilityRoutingResult) (string, error) { +// runUnifiedReAct 执行统一的 ReAct 循环，使用原生 function calling API。 +// messages 数组随交互动态增长：system → history → user → assistant(tool_calls) → tool → ... +// 循环持续到 LLM 返回无 tool_calls 的纯文本回复（即最终回答）或达到安全上限。 +func (o *Orchestrator) runUnifiedReAct(ctx context.Context, chatID, userID, compressedContext, userInput string, fileCtx filePromptContext, appendFileIDText bool) (string, error) { traceID := logger.TraceIDFromContext(ctx) traceLogPrefix := "trace_id=" + traceID - if strings.TrimSpace(routeInput) == "" { - routeInput = composeRouteInput(userInput, fileCtx.Summary) - } - systemPrompt := o.buildUnifiedSystemPrompt(routeInput, route) + systemPrompt := o.buildUnifiedSystemPrompt(userInput) if o.log != nil { - o.log.Infof("%s unified react start route_need_skills=%v route_tools=%v route_skills=%d fallback=%v", traceLogPrefix, route.NeedSkills, route.SelectedToolNames, len(route.SelectedSkills), route.UsedFallback) + o.log.Infof("%s unified react start", traceLogPrefix) } - // 安全上限：防止无限循环（当前暂不使用 reactMaxStep 配置约束，使用固定硬上限） + // 检查 LLM 客户端是否支持原生 tool_calls + toolCallClient, supportsToolCalls := o.llm.(llm.ToolCallChatClient) + if !supportsToolCalls { + if o.log != nil { + o.log.Warnf("%s llm client does not support ToolCallChatClient, falling back to legacy ReAct", traceLogPrefix) + } + return o.runLegacyReAct(ctx, chatID, userID, compressedContext, userInput, fileCtx, appendFileIDText) + } + + // 构建初始 messages 数组 + messages := make([]llm.PromptMessage, 0, 32) + messages = append(messages, llm.PromptMessage{Role: "system", Content: systemPrompt}) + + // 加入历史会话上下文 + //messages = append(messages, parseCompressedHistoryMessages(compressedContext)...) + + // 加入当前用户消息 + messages = append(messages, llm.PromptMessage{Role: "user", Content: userInput}) + + // 构建工具定义列表（通过 API tools 字段传递） + toolDefs := o.buildToolDefinitions() + + const maxSteps = 20 + for step := 1; step <= maxSteps; step++ { + if o.log != nil { + o.log.Infof("%s react step=%d start messages_count=%d", traceLogPrefix, step, len(messages)) + } + + // 调用 LLM（传入完整 messages + tools 定义） + completion, err := toolCallClient.GenerateWithTools(ctx, messages, toolDefs, fileCtx.FileIDs, appendFileIDText) + if err != nil { + return "", err + } + + if o.log != nil { + o.log.Infof("%s react step=%d content_len=%d tool_calls=%d", + traceLogPrefix, step, len(completion.Content), len(completion.ToolCalls)) + if completion.Content != "" { + o.log.Debugf("%s react step=%d thought=%q", traceLogPrefix, step, completion.Content) + } + } + + // ========== 无 tool_calls → 最终回答 ========== + if len(completion.ToolCalls) == 0 { + finalText := strings.TrimSpace(completion.Content) + if finalText == "" { + finalText = "已完成处理。" + } + if o.log != nil { + o.log.Infof("%s react final at step=%d answer_len=%d", traceLogPrefix, step, len(finalText)) + } + return finalText, nil + } + + // ========== 有 tool_calls → 将 assistant 消息加入历史，然后执行工具 ========== + assistantMsg := llm.PromptMessage{ + Role: "assistant", + Content: completion.Content, + ToolCalls: completion.ToolCalls, + } + messages = append(messages, assistantMsg) + + // 逐个执行工具调用，并将结果作为 tool 角色消息加入 + for _, tc := range completion.ToolCalls { + toolName := strings.ToLower(strings.TrimSpace(tc.Function.Name)) + toolInput := extractToolInput(tc.Function.Arguments) + + tool, ok := o.tools.Get(toolName) + if !ok { + if o.log != nil { + o.log.Warnf("%s react step=%d tool_not_found=%s", traceLogPrefix, step, toolName) + } + messages = append(messages, llm.PromptMessage{ + Role: "tool", + ToolCallID: tc.ID, + Name: tc.Function.Name, + Content: formatToolErrorObservation("TOOL_NOT_FOUND", toolName, "该工具不存在，请检查工具名称后重试"), + }) + o.emitCapabilityGap(chatID, userID, userInput, "tool_not_found:"+toolName) + continue + } + + if o.log != nil { + o.log.Infof("%s react step=%d tool_call tool=%s input=%q", traceLogPrefix, step, toolName, toolInput) + } + + toolOut, toolErr := tool.Call(ctx, toolInput) + obs := strings.TrimSpace(toolOut) + if obs == "" { + obs = "(empty output)" + } + if toolErr != nil { + obs = formatToolErrorObservation("TOOL_EXEC_ERROR", toolName, toolErr.Error()) + "\nOUTPUT:\n" + obs + o.emitCapabilityGap(chatID, userID, userInput, "tool_call_failed:"+toolName) + } + // 限制观察值长度防止超出 LLM 上下文窗口 + if len(obs) > 4000 { + obs = obs[:4000] + "\n...(truncated)" + } + + if o.log != nil { + o.log.Infof("%s react step=%d tool=%s observation_len=%d", traceLogPrefix, step, toolName, len(obs)) + } + + messages = append(messages, llm.PromptMessage{ + Role: "tool", + ToolCallID: tc.ID, + Name: tc.Function.Name, + Content: obs, + }) + } + } + + // 达到安全上限仍未得到最终回答 + o.emitCapabilityGap(chatID, userID, userInput, "react_step_exhausted") + return "我尝试了多轮推理与工具调用，但仍未得到稳定结论。请给我更具体的约束或允许我继续尝试。", nil +} + +// runLegacyReAct 是旧版基于 JSON 决策解析的 ReAct 循环，作为不支持 tool_calls 的 LLM 的降级方案。 +func (o *Orchestrator) runLegacyReAct(ctx context.Context, chatID, userID, compressedContext, userInput string, fileCtx filePromptContext, appendFileIDText bool) (string, error) { + traceID := logger.TraceIDFromContext(ctx) + traceLogPrefix := "trace_id=" + traceID + + systemPrompt := o.buildLegacySystemPrompt(userInput) + const maxSteps = 20 scratchpad := "" for step := 1; step <= maxSteps; step++ { if o.log != nil { - o.log.Infof("%s react step=%d start", traceLogPrefix, step) - o.log.Debugf("%s react step=%d scratchpad=%q", traceLogPrefix, step, scratchpad) + o.log.Infof("%s legacy react step=%d start", traceLogPrefix, step) } - // 构造本轮 user prompt：历史上下文 + 用户问题 + 推理记录 - prompt := strings.Join([]string{ - "历史上下文：", - compressedContext, - "", - "用户问题：", - userInput, - "", - "文件上下文：", - defaultIfEmpty(fileCtx.Summary, "(none)"), - "", - "当前推理记录（按时间顺序）：", - scratchpad, - "", - "请输出你的 JSON 决策。", - }, "\n") - - raw, err := o.generateWithOptionalFiles(ctx, systemPrompt, prompt, fileCtx.FileIDs) + messages := buildReActMessages(systemPrompt, compressedContext, userInput, fileCtx.Summary, scratchpad) + raw, err := o.generateWithOptionalFilesMessages(ctx, messages, fileCtx.FileIDs, appendFileIDText) if err != nil { return "", err } - if o.log != nil { - o.log.Infof("%s react step=%d llm_raw=%q", traceLogPrefix, step, raw) - } - // 解析 LLM 返回的 JSON 决策 decision, err := parseDecision(raw) if err != nil { - if o.log != nil { - o.log.Warnf("%s react step=%d parse failed err=%v, using raw as final answer", traceLogPrefix, step, err) - } - // 解析失败时，尝试将原始输出当作直接回答返回 o.emitCapabilityGap(chatID, userID, userInput, "react_parse_failed") return strings.TrimSpace(raw), nil } - if o.log != nil { - o.log.Infof("%s react step=%d thought=%q action=%q is_final=%v", - traceLogPrefix, step, decision.Thought, decision.Action, decision.IsFinalAnswer) - } - - // ========== 判定：是否为最终回答 ========== if decision.IsFinalAnswer { finalText := "" if decision.FinalAnswer != nil { @@ -389,40 +464,26 @@ func (o *Orchestrator) runUnifiedReAct(ctx context.Context, chatID, userID, comp if finalText == "" { finalText = "已完成处理。" } - if o.log != nil { - o.log.Infof("%s react final at step=%d answer=%q", traceLogPrefix, step, finalText) - } return finalText, nil } - // ========== 非最终回答：执行工具调用 ========== action := strings.ToLower(strings.TrimSpace(decision.Action)) if action == "" || action == "none" { - // LLM 说不是最终回答但也不指定工具，记录后让它再想一轮 scratchpad += "Step " + strconv.Itoa(step) + " Thought: " + decision.Thought + "\n" - scratchpad += "Step " + strconv.Itoa(step) + " Observation: 你没有指定要调用的工具，请重新决策：要么调用工具，要么给出最终回答。\n" + scratchpad += "Step " + strconv.Itoa(step) + " Observation: 你没有指定要调用的工具，请重新决策。\n" continue } actionInput := decision.GetActionInputString() - - // 检查工具是否存在 tool, ok := o.tools.Get(action) if !ok { - if o.log != nil { - o.log.Warnf("%s react step=%d tool_not_found=%s", traceLogPrefix, step, action) - } scratchpad += "Step " + strconv.Itoa(step) + " Thought: " + decision.Thought + "\n" scratchpad += "Step " + strconv.Itoa(step) + " Action: " + action + "\n" - scratchpad += "Step " + strconv.Itoa(step) + " Observation: " + formatToolErrorObservation("TOOL_NOT_FOUND", action, "该工具不存在，可用工具请参阅 system prompt") + "\n" + scratchpad += "Step " + strconv.Itoa(step) + " Observation: " + formatToolErrorObservation("TOOL_NOT_FOUND", action, "该工具不存在") + "\n" o.emitCapabilityGap(chatID, userID, userInput, "tool_not_found:"+action) continue } - // 调用工具 - if o.log != nil { - o.log.Infof("%s react step=%d tool_call tool=%s input=%q", traceLogPrefix, step, action, actionInput) - } toolOut, toolErr := tool.Call(ctx, actionInput) obs := strings.TrimSpace(toolOut) if obs == "" { @@ -432,37 +493,95 @@ func (o *Orchestrator) runUnifiedReAct(ctx context.Context, chatID, userID, comp obs = formatToolErrorObservation("TOOL_EXEC_ERROR", action, toolErr.Error()) + "\nOUTPUT:\n" + obs o.emitCapabilityGap(chatID, userID, userInput, "tool_call_failed:"+action) } - // 限制观察值长度防止超出 LLM 上下文窗口 if len(obs) > 4000 { obs = obs[:4000] + "\n...(truncated)" } - if o.log != nil { - o.log.Infof("%s react step=%d observation_len=%d", traceLogPrefix, step, len(obs)) - } - - // 将本轮的思考、行动、观察追加到 scratchpad scratchpad += "Step " + strconv.Itoa(step) + " Thought: " + decision.Thought + "\n" scratchpad += "Step " + strconv.Itoa(step) + " Action: " + action + "\n" scratchpad += "Step " + strconv.Itoa(step) + " ActionInput: " + actionInput + "\n" scratchpad += "Step " + strconv.Itoa(step) + " Observation: " + obs + "\n" } - // 达到安全上限仍未得到最终回答 o.emitCapabilityGap(chatID, userID, userInput, "react_step_exhausted") return "我尝试了多轮推理与工具调用，但仍未得到稳定结论。请给我更具体的约束或允许我继续尝试。", nil } -func composeRouteInput(userInput, fileSummary string) string { - userInput = strings.TrimSpace(userInput) - fileSummary = strings.TrimSpace(fileSummary) - if userInput == "" { - return fileSummary +// buildLegacySystemPrompt 为不支持 tool_calls 的旧版 ReAct 链路构建 system prompt（含 JSON 输出格式约束）。 +func (o *Orchestrator) buildLegacySystemPrompt(userInput string) string { + skillMetaDoc := o.formatSkillSummariesForPrompt() + relevantSkillsDoc := o.formatSelectedSkillsForPrompt(userInput, nil) + toolDoc := o.formatToolDoc() + runtimeDoc := formatRuntimeContextForPrompt() + + return strings.Join([]string{ + "你是一个个人自动化助手，必须遵循如下人格设定并保持一致：", + o.soul, + "", + "===== 运行环境 =====", + runtimeDoc, + "", + "===== 可用技能概览 =====", + skillMetaDoc, + "", + "===== 本轮相关技能 =====", + relevantSkillsDoc, + "", + "===== 可用工具 =====", + toolDoc, + "", + "===== 输出格式约束 =====", + "你必须使用 ReAct 模式进行决策。每次回复必须是且仅是一个 JSON 对象：", + "{", + " \"thought\": \"你的推理过程（必填）\",", + " \"action\": \"要调用的工具名称（不调工具时填 none）\",", + " \"action_input\": \"传给工具的输入\",", + " \"is_final_answer\": true 或 false,", + " \"final_answer\": \"当 is_final_answer=true 时填写给用户的最终回复\"", + "}", + }, "\n") +} + +// buildToolDefinitions 将工具注册表转换为 OpenAI function calling 所需的 ToolDefinition 列表。 +func (o *Orchestrator) buildToolDefinitions() []llm.ToolDefinition { + list := o.tools.List() + defs := make([]llm.ToolDefinition, 0, len(list)) + defaultParams := json.RawMessage(`{"type":"object","properties":{"input":{"type":"string","description":"工具的输入命令或查询内容"}},"required":["input"]}`) + + sort.Slice(list, func(i, j int) bool { + return list[i].Name() < list[j].Name() + }) + + for _, t := range list { + defs = append(defs, llm.ToolDefinition{ + Type: "function", + Function: llm.ToolFunctionDef{ + Name: t.Name(), + Description: t.Description(), + Parameters: defaultParams, + }, + }) } - if fileSummary == "" { - return userInput + return defs +} + +// extractToolInput 从 LLM 的 function calling arguments JSON 中提取工具输入字符串。 +func extractToolInput(arguments string) string { + arguments = strings.TrimSpace(arguments) + if arguments == "" { + return "" } - return userInput + "\n\n" + fileSummary + var args struct { + Input string `json:"input"` + } + if err := json.Unmarshal([]byte(arguments), &args); err != nil { + // 降级：直接将 arguments 作为输入 + return arguments + } + if args.Input != "" { + return args.Input + } + return arguments } func (o *Orchestrator) prepareFilePromptContext(ctx context.Context, files []llm.InputFile, pending []pendingFileRef) filePromptContext { @@ -535,16 +654,85 @@ func buildFileSummary(pending, uploaded []pendingFileRef) string { return strings.Join(lines, "\n") } -func (o *Orchestrator) generateWithOptionalFiles(ctx context.Context, systemPrompt, userPrompt string, fileIDs []string) (string, error) { +func (o *Orchestrator) generateWithOptionalFilesMessages(ctx context.Context, messages []llm.PromptMessage, fileIDs []string, appendFileIDText bool) (string, error) { ids := nonEmptyIDs(fileIDs) if len(ids) == 0 { + if client, ok := o.llm.(llm.MessageChatClient); ok { + return client.GenerateMessages(ctx, messages) + } + systemPrompt, userPrompt := fallbackPromptsFromMessages(messages) return o.llm.Generate(ctx, systemPrompt, userPrompt) } + if client, ok := o.llm.(llm.FileMessageChatClient); ok { + return client.GenerateMessagesWithFiles(ctx, messages, ids, appendFileIDText) + } client, ok := o.llm.(llm.FileChatClient) if !ok { + systemPrompt, userPrompt := fallbackPromptsFromMessages(messages) return o.llm.Generate(ctx, systemPrompt, userPrompt) } - return client.GenerateWithFiles(ctx, systemPrompt, userPrompt, ids) + systemPrompt, userPrompt := fallbackPromptsFromMessages(messages) + return client.GenerateWithFiles(ctx, systemPrompt, userPrompt, ids, appendFileIDText) +} + +func buildReActMessages(systemPrompt, compressedContext, userInput, fileSummary, scratchpad string) []llm.PromptMessage { + msgs := make([]llm.PromptMessage, 0, 16) + msgs = append(msgs, llm.PromptMessage{Role: "system", Content: systemPrompt}) + msgs = append(msgs, parseCompressedHistoryMessages(compressedContext)...) + + if strings.TrimSpace(fileSummary) != "" { + msgs = append(msgs, llm.PromptMessage{Role: "assistant", Content: "文件上下文摘要:\n" + strings.TrimSpace(fileSummary)}) + } + if strings.TrimSpace(scratchpad) != "" { + msgs = append(msgs, llm.PromptMessage{Role: "assistant", Content: "推理记录:\n" + strings.TrimSpace(scratchpad)}) + } + msgs = append(msgs, llm.PromptMessage{Role: "user", Content: userInput}) + return msgs +} + +func parseCompressedHistoryMessages(compressed string) []llm.PromptMessage { + compressed = strings.TrimSpace(compressed) + if compressed == "" { + return nil + } + lines := strings.Split(compressed, "\n") + out := make([]llm.PromptMessage, 0, len(lines)) + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + idx := strings.Index(line, ":") + if idx <= 0 { + out = append(out, llm.PromptMessage{Role: "assistant", Content: line}) + continue + } + role := strings.ToLower(strings.TrimSpace(line[:idx])) + content := strings.TrimSpace(line[idx+1:]) + if role != "system" && role != "user" && role != "assistant" { + role = "assistant" + } + out = append(out, llm.PromptMessage{Role: role, Content: content}) + } + return out +} + +func fallbackPromptsFromMessages(messages []llm.PromptMessage) (string, string) { + sysParts := make([]string, 0, 2) + userParts := make([]string, 0, len(messages)) + for _, m := range messages { + role := strings.ToLower(strings.TrimSpace(m.Role)) + content := strings.TrimSpace(m.Content) + if content == "" { + continue + } + if role == "system" { + sysParts = append(sysParts, content) + continue + } + userParts = append(userParts, role+": "+content) + } + return strings.Join(sysParts, "\n\n"), strings.Join(userParts, "\n") } func (o *Orchestrator) buildFileUploadAck(ctx filePromptContext) string { @@ -670,180 +858,6 @@ func (o *Orchestrator) formatSelectedSkillsForPrompt(userInput string, selected return formatSkills(skills) } -func (o *Orchestrator) routeCapabilities(ctx context.Context, userInput string) capabilityRoutingResult { - fallback := capabilityRoutingResult{ - NeedSkills: true, - SelectedSkills: o.selectRelevantSkills(userInput, 4), - Reason: "router fallback: keyword matching", - UsedFallback: true, - } - - raw, err := o.llm.Generate(ctx, o.buildRouteSystemPrompt(), o.buildRouteUserPrompt(userInput)) - if err != nil { - if o.log != nil { - o.log.Warnf("capability router llm call failed err=%v", err) - } - return fallback - } - - decision, err := parseCapabilityRoute(raw) - if err != nil { - if o.log != nil { - o.log.Warnf("capability router parse failed err=%v raw=%q", err, raw) - } - return fallback - } - - resolvedTools := o.normalizeToolSelection(decision.SelectedTools) - resolved := capabilityRoutingResult{ - NeedSkills: decision.NeedSkills, - SelectedToolNames: resolvedTools, - Reason: strings.TrimSpace(decision.Reason), - } - - if resolved.NeedSkills { - skills := o.resolveSkillsByNames(decision.SelectedSkills, 4) - if len(skills) == 0 { - skills = o.selectRelevantSkills(userInput, 4) - resolved.UsedFallback = true - } - resolved.SelectedSkills = skills - } - - return resolved -} - -func (o *Orchestrator) buildRouteSystemPrompt() string { - return strings.Join([]string{ - "你是能力路由器（Router Agent）。", - "你的任务是：在不加载技能全文的前提下，仅根据工具摘要和技能摘要，判断本请求是否可以仅靠原子工具能力完成，还是需要加载技能详细说明。", - "输出必须且仅能是 JSON：", - "{", - " \"need_skills\": true 或 false,", - " \"selected_tools\": [\"tool_name\", ...],", - " \"selected_skills\": [\"skill_name\", ...],", - " \"reason\": \"简短路由理由\"", - "}", - "规则：", - "1) 优先原子工具能力。若可通过工具链路完成，need_skills=false。", - "2) 只有当工具能力不足以覆盖业务约束时，need_skills=true 并选择少量最相关技能。", - "3) selected_skills 仅填写技能名称（来自技能摘要）。", - "4) selected_tools 仅填写可用工具名。", - "5) 不要输出 JSON 之外内容。", - }, "\n") -} - -func (o *Orchestrator) buildRouteUserPrompt(userInput string) string { - return strings.Join([]string{ - "当前运行环境：", - formatRuntimeContextForPrompt(), - "", - "用户问题：", - userInput, - "", - "可用工具摘要：", - o.formatToolDoc(), - "", - "可用技能摘要：", - o.formatSkillSummariesForPrompt(), - "", - "请给出路由 JSON。", - }, "\n") -} - -func (o *Orchestrator) normalizeToolSelection(in []string) []string { - if len(in) == 0 { - return nil - } - allowed := map[string]struct{}{} - for _, t := range o.tools.List() { - allowed[strings.ToLower(strings.TrimSpace(t.Name()))] = struct{}{} - } - out := make([]string, 0, len(in)) - set := map[string]struct{}{} - for _, name := range in { - n := strings.ToLower(strings.TrimSpace(name)) - if n == "" { - continue - } - if _, ok := allowed[n]; !ok { - continue - } - if _, exists := set[n]; exists { - continue - } - set[n] = struct{}{} - out = append(out, n) - } - sort.Strings(out) - return out -} - -func (o *Orchestrator) resolveSkillsByNames(names []string, maxCount int) []knowledge.Skill { - if len(names) == 0 { - return nil - } - if maxCount <= 0 { - maxCount = 4 - } - all := o.getSkillsSnapshot() - idx := make(map[string]knowledge.Skill, len(all)) - for _, sk := range all { - key := strings.ToLower(strings.TrimSpace(sk.Name)) - if key != "" { - idx[key] = sk - } - } - out := make([]knowledge.Skill, 0, maxCount) - used := map[string]struct{}{} - for _, name := range names { - key := strings.ToLower(strings.TrimSpace(name)) - if key == "" { - continue - } - sk, ok := idx[key] - if !ok { - continue - } - if _, exists := used[key]; exists { - continue - } - used[key] = struct{}{} - out = append(out, sk) - if len(out) >= maxCount { - break - } - } - return out -} - -func formatRouteForPrompt(route capabilityRoutingResult) string { - b := strings.Builder{} - if route.UsedFallback { - b.WriteString("router_status: fallback\n") - } else { - b.WriteString("router_status: ok\n") - } - b.WriteString("need_skills: ") - b.WriteString(strconv.FormatBool(route.NeedSkills)) - b.WriteString("\n") - b.WriteString("selected_tools: ") - if len(route.SelectedToolNames) == 0 { - b.WriteString("(none)") - } else { - b.WriteString(strings.Join(route.SelectedToolNames, ", ")) - } - b.WriteString("\n") - b.WriteString("selected_skill_count: ") - b.WriteString(strconv.Itoa(len(route.SelectedSkills))) - b.WriteString("\n") - if strings.TrimSpace(route.Reason) != "" { - b.WriteString("reason: ") - b.WriteString(strings.TrimSpace(route.Reason)) - } - return strings.TrimSpace(b.String()) -} - func (o *Orchestrator) selectRelevantSkills(userInput string, maxCount int) []knowledge.Skill { if maxCount <= 0 { maxCount = 4 diff --git a/internal/llm/client.go b/internal/llm/client.go index 2a1e9f4..63fb3df 100644 --- a/internal/llm/client.go +++ b/internal/llm/client.go @@ -5,28 +5,83 @@ import ( "context" "encoding/json" "fmt" - "io" - "mime/multipart" - "net/http" "strings" "time" "laodingbot/internal/config" "laodingbot/internal/logger" + + openai "github.com/openai/openai-go" // imported as openai + "github.com/openai/openai-go/option" + "github.com/openai/openai-go/packages/param" + "github.com/openai/openai-go/shared" ) type Client interface { Generate(ctx context.Context, systemPrompt, userPrompt string) (string, error) } +type PromptMessage struct { + Role string `json:"role"` + Content string `json:"content"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` + Name string `json:"name,omitempty"` +} + +type MessageChatClient interface { + GenerateMessages(ctx context.Context, messages []PromptMessage) (string, error) +} + type FileChatClient interface { - GenerateWithFiles(ctx context.Context, systemPrompt, userPrompt string, fileIDs []string) (string, error) + GenerateWithFiles(ctx context.Context, systemPrompt, userPrompt string, fileIDs []string, appendFileIDText bool) (string, error) +} + +type FileMessageChatClient interface { + GenerateMessagesWithFiles(ctx context.Context, messages []PromptMessage, fileIDs []string, appendFileIDText bool) (string, error) } type FileUploader interface { UploadFile(ctx context.Context, file InputFile, purpose string) (string, error) } +// ToolCallChatClient 支持原生 function calling 的 LLM 客户端接口。 +type ToolCallChatClient interface { + GenerateWithTools(ctx context.Context, messages []PromptMessage, tools []ToolDefinition, fileIDs []string, appendFileIDText bool) (*ChatCompletion, error) +} + +// ToolDefinition 描述一个可供 LLM 调用的工具函数定义。 +type ToolDefinition struct { + Type string `json:"type"` + Function ToolFunctionDef `json:"function"` +} + +// ToolFunctionDef 是工具函数的名称、描述和参数 JSON Schema。 +type ToolFunctionDef struct { + Name string `json:"name"` + Description string `json:"description"` + Parameters json.RawMessage `json:"parameters,omitempty"` +} + +// ToolCall 是 LLM 在响应中返回的工具调用请求。 +type ToolCall struct { + ID string `json:"id"` + Type string `json:"type"` + Function ToolCallFunction `json:"function"` +} + +// ToolCallFunction 包含工具调用的函数名和参数。 +type ToolCallFunction struct { + Name string `json:"name"` + Arguments string `json:"arguments"` +} + +// ChatCompletion 是 LLM 响应的结构化表示，包含文本内容和可选的工具调用。 +type ChatCompletion struct { + Content string + ToolCalls []ToolCall +} + type InputFile struct { FileName string MimeType string @@ -34,206 +89,312 @@ type InputFile struct { } type OpenAICompatibleClient struct { - baseURL string - apiKey string + client openai.Client model string fileModel string filePromptMode string - http *http.Client log *logger.Logger } func NewOpenAICompatibleClient(cfg config.LLMConfig, log *logger.Logger) *OpenAICompatibleClient { + opts := []option.RequestOption{ + option.WithAPIKey(cfg.APIKey), + option.WithRequestTimeout(60 * time.Second), + } + if strings.TrimSpace(cfg.BaseURL) != "" { + opts = append(opts, option.WithBaseURL(cfg.BaseURL)) + } return &OpenAICompatibleClient{ - baseURL: cfg.BaseURL, - apiKey: cfg.APIKey, + client: openai.NewClient(opts...), model: cfg.Model, fileModel: cfg.FileModel, filePromptMode: cfg.FilePromptMode, - http: &http.Client{Timeout: 60 * time.Second}, log: log, } } -type chatRequest struct { - Model string `json:"model"` - Messages []chatMessage `json:"messages"` -} - -type chatMessage struct { - Role string `json:"role"` - Content any `json:"content"` -} - -type chatContentPart struct { - Type string `json:"type"` - Text string `json:"text,omitempty"` - FileID string `json:"file_id,omitempty"` -} - -type chatResponse struct { - Choices []struct { - Message struct { - Role string `json:"role"` - Content string `json:"content"` - } `json:"message"` - } `json:"choices"` - Error *struct { - Message string `json:"message"` - } `json:"error,omitempty"` -} - -type fileUploadResponse struct { - ID string `json:"id"` - Bytes int64 `json:"bytes,omitempty"` - CreatedAt int64 `json:"created_at,omitempty"` - Filename string `json:"filename,omitempty"` - Object string `json:"object,omitempty"` - Purpose string `json:"purpose,omitempty"` - Code int `json:"code,omitempty"` - Message string `json:"message,omitempty"` - Status any `json:"status,omitempty"` - StatusDetails any `json:"status_details,omitempty"` - Data *struct { - ID string `json:"id"` - } `json:"data,omitempty"` - Error *struct { - Message string `json:"message"` - } `json:"error,omitempty"` -} - func (c *OpenAICompatibleClient) Generate(ctx context.Context, systemPrompt, userPrompt string) (string, error) { - return c.generateInternal(ctx, systemPrompt, userPrompt, nil) + messages := []PromptMessage{ + {Role: "system", Content: systemPrompt}, + {Role: "user", Content: userPrompt}, + } + return c.generateWithMessagesInternal(ctx, messages, nil, false) } -func (c *OpenAICompatibleClient) GenerateWithFiles(ctx context.Context, systemPrompt, userPrompt string, fileIDs []string) (string, error) { - return c.generateInternal(ctx, systemPrompt, userPrompt, fileIDs) +func (c *OpenAICompatibleClient) GenerateWithFiles(ctx context.Context, systemPrompt, userPrompt string, fileIDs []string, appendFileIDText bool) (string, error) { + messages := []PromptMessage{ + {Role: "system", Content: systemPrompt}, + {Role: "user", Content: userPrompt}, + } + return c.generateWithMessagesInternal(ctx, messages, fileIDs, appendFileIDText) } -func (c *OpenAICompatibleClient) generateInternal(ctx context.Context, systemPrompt, userPrompt string, fileIDs []string) (string, error) { +func (c *OpenAICompatibleClient) GenerateMessages(ctx context.Context, messages []PromptMessage) (string, error) { + return c.generateWithMessagesInternal(ctx, messages, nil, false) +} + +func (c *OpenAICompatibleClient) GenerateMessagesWithFiles(ctx context.Context, messages []PromptMessage, fileIDs []string, appendFileIDText bool) (string, error) { + return c.generateWithMessagesInternal(ctx, messages, fileIDs, appendFileIDText) +} + +// GenerateWithTools 使用原生 function calling 发送请求，返回结构化的 ChatCompletion。 +func (c *OpenAICompatibleClient) GenerateWithTools(ctx context.Context, messages []PromptMessage, tools []ToolDefinition, fileIDs []string, appendFileIDText bool) (*ChatCompletion, error) { model := c.model ids := nonEmptyIDs(fileIDs) - if len(ids) > 0 { - if strings.TrimSpace(c.fileModel) != "" { - model = c.fileModel - } + if len(ids) > 0 && strings.TrimSpace(c.fileModel) != "" { + model = c.fileModel } + + sdkMessages := buildSDKMessages(messages, ids, c.normalizedFilePromptMode(), appendFileIDText) + sdkTools := toSDKTools(tools) + if c.log != nil { - c.log.Debugf("llm request start model=%s system_len=%d user_len=%d file_count=%d file_prompt_mode=%s", model, len(systemPrompt), len(userPrompt), len(ids), c.normalizedFilePromptMode()) + c.log.Debugf("llm tool-call request start model=%s messages=%d tools=%d files=%d", model, len(sdkMessages), len(sdkTools), len(ids)) } - messages := buildMessages(systemPrompt, userPrompt, ids, c.normalizedFilePromptMode()) - body := chatRequest{ - Model: model, - Messages: messages, + + params := openai.ChatCompletionNewParams{ + Model: shared.ChatModel(model), + Messages: sdkMessages, } - b, err := json.Marshal(body) + if len(sdkTools) > 0 { + params.Tools = sdkTools + } + + if c.log != nil { + if b, err := json.Marshal(params); err == nil { + c.log.Debugf("llm tool-call request params: %s", string(b)) + } + } + + resp, err := c.client.Chat.Completions.New(ctx, params) + if err != nil { + return nil, fmt.Errorf("llm tool-call request failed: %w", err) + } + + if len(resp.Choices) == 0 { + return nil, fmt.Errorf("llm returned empty choices") + } + + choice := resp.Choices[0] + resultToolCalls := fromSDKToolCalls(choice.Message.ToolCalls) + if c.log != nil { + c.log.Infof("llm tool-call response success model=%s content_len=%d tool_calls=%d finish=%s", + model, len(choice.Message.Content), len(resultToolCalls), choice.FinishReason) + } + + return &ChatCompletion{ + Content: choice.Message.Content, + ToolCalls: resultToolCalls, + }, nil +} + +func (c *OpenAICompatibleClient) generateWithMessagesInternal(ctx context.Context, messages []PromptMessage, fileIDs []string, appendFileIDText bool) (string, error) { + model := c.model + ids := nonEmptyIDs(fileIDs) + if len(ids) > 0 && strings.TrimSpace(c.fileModel) != "" { + model = c.fileModel + } + + baseMessages := normalizePromptMessages(messages) + if len(baseMessages) == 0 { + baseMessages = []PromptMessage{{Role: "user", Content: ""}} + } + + systemLen, userLen := promptMessageLengths(baseMessages) + if c.log != nil { + c.log.Debugf("llm request start model=%s system_len=%d user_len=%d file_count=%d file_prompt_mode=%s", model, systemLen, userLen, len(ids), c.normalizedFilePromptMode()) + } + + sdkMessages := buildSDKMessages(baseMessages, ids, c.normalizedFilePromptMode(), appendFileIDText) + + params := openai.ChatCompletionNewParams{ + Model: shared.ChatModel(model), + Messages: sdkMessages, + } + + resp, err := c.client.Chat.Completions.New(ctx, params) if err != nil { if c.log != nil { - c.log.Errorf("marshal llm request failed err=%v", err) + c.log.Errorf("llm request failed err=%v", err) } - return "", err + return "", fmt.Errorf("llm request failed: %w", err) } - url := strings.TrimRight(c.baseURL, "/") + "/chat/completions" - req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(b)) - if err != nil { + if len(resp.Choices) == 0 { if c.log != nil { - c.log.Errorf("build llm request failed err=%v", err) - } - return "", err - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+c.apiKey) - - resp, err := c.http.Do(req) - if err != nil { - if c.log != nil { - c.log.Errorf("llm http request failed err=%v", err) - } - return "", err - } - defer resp.Body.Close() - - raw, err := io.ReadAll(resp.Body) - if err != nil { - if c.log != nil { - c.log.Errorf("llm read response failed err=%v", err) - } - return "", err - } - - var out chatResponse - if err := json.Unmarshal(raw, &out); err != nil { - if c.log != nil { - c.log.Errorf("llm response unmarshal failed status=%d err=%v", resp.StatusCode, err) - } - return "", err - } - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - if c.log != nil { - c.log.Errorf("llm bad status=%d", resp.StatusCode) - } - if out.Error != nil && out.Error.Message != "" { - return "", fmt.Errorf("llm error: %s", out.Error.Message) - } - return "", fmt.Errorf("llm error status: %d", resp.StatusCode) - } - - if len(out.Choices) == 0 { - if c.log != nil { - c.log.Errorf("llm returned empty choices status=%d", resp.StatusCode) + c.log.Errorf("llm returned empty choices") } return "", fmt.Errorf("llm returned empty choices") } + + content := resp.Choices[0].Message.Content if c.log != nil { - c.log.Infof("llm response success model=%s output_len=%d", model, len(out.Choices[0].Message.Content)) + c.log.Infof("llm response success model=%s output_len=%d", model, len(content)) } - - return out.Choices[0].Message.Content, nil + return content, nil } -func buildMessages(systemPrompt, userPrompt string, fileIDs []string, mode string) []chatMessage { +// buildSDKMessages 将 PromptMessage 列表转换为 openai SDK 的消息格式，并注入 file_id（如需要）。 +func buildSDKMessages(base []PromptMessage, fileIDs []string, mode string, appendFileIDText bool) []openai.ChatCompletionMessageParamUnion { mode = strings.ToLower(strings.TrimSpace(mode)) - if mode == "system_fileid_uri" { - msgs := []chatMessage{{Role: "system", Content: systemPrompt}} - for _, id := range fileIDs { - if strings.TrimSpace(id) == "" { - continue - } - msgs = append(msgs, chatMessage{Role: "system", Content: "fileid://" + strings.TrimSpace(id)}) + out := make([]openai.ChatCompletionMessageParamUnion, 0, len(base)+2) + + for _, m := range base { + role := normalizeRole(m.Role) + if role == "" { + continue } - msgs = append(msgs, chatMessage{Role: "user", Content: userPrompt}) - return msgs + out = append(out, toSDKMessage(m, role)) } - userContent := buildUserContent(userPrompt, fileIDs) - return []chatMessage{ - {Role: "system", Content: systemPrompt}, - {Role: "user", Content: userContent}, + + if len(fileIDs) == 0 { + return out + } + + if appendFileIDText { + // WebUI 场景：将首个 fileID 作为 text part 追加到最后一个 user 消息。 + firstFileID := strings.TrimSpace(fileIDs[0]) + if firstFileID == "" { + return out + } + for i := len(out) - 1; i >= 0; i-- { + if r := out[i].GetRole(); r != nil && *r == "user" { + out[i] = buildUserMessageWithFileIDText(out[i], firstFileID) + return out + } + } + out = append(out, buildUserMessageWithFileIDText(openai.UserMessage(""), firstFileID)) + return out + } + + // 非 WebUI 场景：保持原有 file content part 方式。 + for i := len(out) - 1; i >= 0; i-- { + if r := out[i].GetRole(); r != nil && *r == "user" { + out[i] = buildUserMessageWithFiles(out[i], fileIDs) + return out + } + } + out = append(out, buildUserMessageWithFiles(openai.UserMessage(""), fileIDs)) + return out +} + +// toSDKMessage 将单个 PromptMessage 转换为 openai SDK 消息类型。 +func toSDKMessage(m PromptMessage, role string) openai.ChatCompletionMessageParamUnion { + switch role { + case "system": + return openai.SystemMessage(m.Content) + case "user": + return openai.UserMessage(m.Content) + case "assistant": + if len(m.ToolCalls) > 0 { + sdkToolCalls := make([]openai.ChatCompletionMessageToolCallParam, 0, len(m.ToolCalls)) + for _, tc := range m.ToolCalls { + sdkToolCalls = append(sdkToolCalls, openai.ChatCompletionMessageToolCallParam{ + ID: tc.ID, + Function: openai.ChatCompletionMessageToolCallFunctionParam{ + Name: tc.Function.Name, + Arguments: tc.Function.Arguments, + }, + }) + } + msg := openai.AssistantMessage(m.Content) + msg.OfAssistant.ToolCalls = sdkToolCalls + return msg + } + return openai.AssistantMessage(m.Content) + case "tool": + return openai.ToolMessage(m.Content, m.ToolCallID) + default: + return openai.UserMessage(m.Content) } } -func buildUserContent(userPrompt string, fileIDs []string) any { - trimmedPrompt := strings.TrimSpace(userPrompt) - if len(fileIDs) == 0 { - return userPrompt +// buildUserMessageWithFileIDText 为 user 消息追加一个 text part，内容为 fileID。 +func buildUserMessageWithFileIDText(msg openai.ChatCompletionMessageParamUnion, fileID string) openai.ChatCompletionMessageParamUnion { + // 提取已有的文本内容 + text := "" + if s, ok := msg.GetContent().AsAny().(*string); ok && s != nil { + text = *s + } + fileID = strings.TrimSpace(fileID) + if fileID == "" { + return msg } - parts := make([]chatContentPart, 0, len(fileIDs)+1) - if trimmedPrompt != "" { - parts = append(parts, chatContentPart{Type: "text", Text: userPrompt}) + parts := make([]openai.ChatCompletionContentPartUnionParam, 0, 2) + if strings.TrimSpace(text) != "" { + parts = append(parts, openai.TextContentPart(text)) + } + parts = append(parts, openai.TextContentPart(fileID)) + if len(parts) == 0 { + return msg + } + return openai.UserMessage(parts) +} + +// buildUserMessageWithFiles 为 user 消息追加 file content parts。 +func buildUserMessageWithFiles(msg openai.ChatCompletionMessageParamUnion, fileIDs []string) openai.ChatCompletionMessageParamUnion { + text := "" + if s, ok := msg.GetContent().AsAny().(*string); ok && s != nil { + text = *s + } + + parts := make([]openai.ChatCompletionContentPartUnionParam, 0, len(fileIDs)+1) + if strings.TrimSpace(text) != "" { + parts = append(parts, openai.TextContentPart(text)) } for _, id := range fileIDs { id = strings.TrimSpace(id) if id == "" { continue } - parts = append(parts, chatContentPart{Type: "file", FileID: id}) + parts = append(parts, openai.FileContentPart(openai.ChatCompletionContentPartFileFileParam{FileID: param.NewOpt(id)})) } if len(parts) == 0 { - return userPrompt + return msg } - return parts + return openai.UserMessage(parts) +} + +// toSDKTools 将内部 ToolDefinition 列表转换为 openai SDK 的 ChatCompletionToolParam 列表。 +func toSDKTools(tools []ToolDefinition) []openai.ChatCompletionToolParam { + if len(tools) == 0 { + return nil + } + out := make([]openai.ChatCompletionToolParam, 0, len(tools)) + for _, t := range tools { + var params shared.FunctionParameters + if len(t.Function.Parameters) > 0 { + _ = json.Unmarshal(t.Function.Parameters, ¶ms) + } + out = append(out, openai.ChatCompletionToolParam{ + Function: shared.FunctionDefinitionParam{ + Name: t.Function.Name, + Description: param.NewOpt(t.Function.Description), + Parameters: params, + }, + }) + } + return out +} + +// fromSDKToolCalls 将 openai SDK 响应中的 tool calls 转换为内部 ToolCall 类型。 +func fromSDKToolCalls(sdkCalls []openai.ChatCompletionMessageToolCall) []ToolCall { + if len(sdkCalls) == 0 { + return nil + } + out := make([]ToolCall, 0, len(sdkCalls)) + for _, tc := range sdkCalls { + out = append(out, ToolCall{ + ID: tc.ID, + Type: "function", + Function: ToolCallFunction{ + Name: tc.Function.Name, + Arguments: tc.Function.Arguments, + }, + }) + } + return out } func (c *OpenAICompatibleClient) UploadFile(ctx context.Context, file InputFile, purpose string) (string, error) { @@ -248,7 +409,6 @@ func (c *OpenAICompatibleClient) UploadFile(ctx context.Context, file InputFile, if purpose != "" { purposes = append(purposes, purpose) } - // Provider compatibility fallback order. purposes = appendIfMissing(purposes, "file-extract") purposes = appendIfMissing(purposes, "batch") @@ -270,77 +430,24 @@ func (c *OpenAICompatibleClient) UploadFile(ctx context.Context, file InputFile, } func (c *OpenAICompatibleClient) uploadFileOnce(ctx context.Context, file InputFile, purpose string) (string, error) { - - body := &bytes.Buffer{} - writer := multipart.NewWriter(body) - if err := writer.WriteField("purpose", purpose); err != nil { - return "", err - } - part, err := writer.CreateFormFile("file", file.FileName) + resp, err := c.client.Files.New(ctx, openai.FileNewParams{ + File: bytes.NewReader(file.Content), + Purpose: openai.FilePurpose(purpose), + }) if err != nil { - return "", err - } - if _, err := part.Write(file.Content); err != nil { - return "", err - } - if err := writer.Close(); err != nil { - return "", err + return "", fmt.Errorf("llm file upload failed: %w", err) } - url := strings.TrimRight(c.baseURL, "/") + "/files" - req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, body) - if err != nil { - return "", err - } - req.Header.Set("Content-Type", writer.FormDataContentType()) - req.Header.Set("Authorization", "Bearer "+c.apiKey) - - resp, err := c.http.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - - raw, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - - var out fileUploadResponse - if err := json.Unmarshal(raw, &out); err != nil { - return "", fmt.Errorf("llm file upload response decode failed: %w body=%s", err, clipForError(raw)) - } - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - if strings.TrimSpace(out.Message) != "" { - return "", fmt.Errorf("llm file upload error: %s", out.Message) - } - if out.Error != nil && out.Error.Message != "" { - return "", fmt.Errorf("llm file upload error: %s", out.Error.Message) - } - return "", fmt.Errorf("llm file upload status: %d body=%s", resp.StatusCode, clipForError(raw)) - } - fileID := strings.TrimSpace(out.ID) - if fileID == "" && out.Data != nil { - fileID = strings.TrimSpace(out.Data.ID) - } + fileID := strings.TrimSpace(resp.ID) if fileID == "" { - return "", fmt.Errorf("llm file upload returned empty file id body=%s", clipForError(raw)) + return "", fmt.Errorf("llm file upload returned empty file id") } if c.log != nil { - c.log.Infof("llm file uploaded name=%s size=%d file_id=%s purpose=%s status=%v", file.FileName, len(file.Content), fileID, purpose, out.Status) + c.log.Infof("llm file uploaded name=%s size=%d file_id=%s purpose=%s", file.FileName, len(file.Content), fileID, purpose) } return fileID, nil } -func clipForError(raw []byte) string { - s := strings.TrimSpace(string(raw)) - const max = 400 - if len(s) <= max { - return s - } - return s[:max] + "...(truncated)" -} - func appendIfMissing(items []string, value string) []string { value = strings.TrimSpace(value) if value == "" { @@ -374,6 +481,46 @@ func nonEmptyIDs(ids []string) []string { return out } +func normalizePromptMessages(messages []PromptMessage) []PromptMessage { + out := make([]PromptMessage, 0, len(messages)) + for _, m := range messages { + role := normalizeRole(m.Role) + if role == "" { + continue + } + out = append(out, PromptMessage{ + Role: role, + Content: m.Content, + ToolCalls: m.ToolCalls, + ToolCallID: m.ToolCallID, + Name: m.Name, + }) + } + return out +} + +func normalizeRole(role string) string { + r := strings.ToLower(strings.TrimSpace(role)) + if r != "system" && r != "user" && r != "assistant" && r != "tool" { + return "" + } + return r +} + +func promptMessageLengths(messages []PromptMessage) (int, int) { + systemLen := 0 + userLen := 0 + for _, m := range messages { + switch normalizeRole(m.Role) { + case "system": + systemLen += len(m.Content) + case "user": + userLen += len(m.Content) + } + } + return systemLen, userLen +} + func (c *OpenAICompatibleClient) normalizedFilePromptMode() string { mode := strings.ToLower(strings.TrimSpace(c.filePromptMode)) if mode == "system_fileid" || mode == "system_fileid_url" || mode == "system_fileid_uri" { diff --git a/internal/transport/webui/bot.go b/internal/transport/webui/bot.go index 6bfe87d..c85aed7 100644 --- a/internal/transport/webui/bot.go +++ b/internal/transport/webui/bot.go @@ -18,9 +18,10 @@ import ( ) type IncomingMessage struct { - ChatID string - UserID string - Text string + ChatID string + UserID string + Text string + FileIDs []string } type ChatHandler func(context.Context, IncomingMessage) (string, error) @@ -37,9 +38,41 @@ type Bot struct { } type chatRequest struct { - Text string `json:"text"` - SessionID string `json:"session_id"` - UserID string `json:"user_id"` + Text string `json:"text"` + SessionID string `json:"session_id"` + UserID string `json:"user_id"` + FileIDs []string `json:"file_ids"` +} + +func (r *chatRequest) UnmarshalJSON(data []byte) error { + type rawChatRequest struct { + Text string `json:"text"` + SessionID string `json:"session_id"` + SessionIDCamel string `json:"sessionId"` + UserID string `json:"user_id"` + UserIDCamel string `json:"userId"` + FileIDs json.RawMessage `json:"file_ids"` + FileIDsCamel json.RawMessage `json:"fileIds"` + FileIDsFlat json.RawMessage `json:"fileids"` + FileID json.RawMessage `json:"file_id"` + } + + var raw rawChatRequest + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + + r.Text = raw.Text + r.SessionID = firstNonEmpty(raw.SessionID, raw.SessionIDCamel) + r.UserID = firstNonEmpty(raw.UserID, raw.UserIDCamel) + + rawIDs := firstNonEmptyRaw(raw.FileIDs, raw.FileIDsCamel, raw.FileIDsFlat, raw.FileID) + ids, err := decodeStringList(rawIDs) + if err != nil { + return err + } + r.FileIDs = ids + return nil } type chatResponse struct { @@ -158,9 +191,10 @@ func (b *Bot) handleChat(w http.ResponseWriter, r *http.Request) { userID := b.resolveID(req.UserID, "user") reply, err := b.chatHandler(r.Context(), IncomingMessage{ - ChatID: sessionID, - UserID: userID, - Text: req.Text, + ChatID: sessionID, + UserID: userID, + Text: req.Text, + FileIDs: req.FileIDs, }) if err != nil { if b.log != nil { @@ -176,6 +210,65 @@ func (b *Bot) handleChat(w http.ResponseWriter, r *http.Request) { }) } +func decodeStringList(raw json.RawMessage) ([]string, error) { + if len(raw) == 0 { + return nil, nil + } + + var list []string + if err := json.Unmarshal(raw, &list); err == nil { + return nonEmptyIDs(list), nil + } + + var single string + if err := json.Unmarshal(raw, &single); err == nil { + if strings.TrimSpace(single) == "" { + return nil, nil + } + return nonEmptyIDs(strings.Split(single, ",")), nil + } + + return nil, fmt.Errorf("invalid file ids format") +} + +func firstNonEmptyRaw(vals ...json.RawMessage) json.RawMessage { + for _, v := range vals { + if len(v) > 0 { + return v + } + } + return nil +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if strings.TrimSpace(v) != "" { + return v + } + } + return "" +} + +func nonEmptyIDs(ids []string) []string { + if len(ids) == 0 { + return nil + } + out := make([]string, 0, len(ids)) + seen := map[string]struct{}{} + for _, id := range ids { + id = strings.TrimSpace(id) + if id == "" { + continue + } + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + out = append(out, id) + } + return out +} + func (b *Bot) handleUpload(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { writeJSON(w, http.StatusMethodNotAllowed, errorResponse{Error: "method not allowed"}) diff --git a/internal/transport/webui/bot_test.go b/internal/transport/webui/bot_test.go index bd01469..42c5378 100644 --- a/internal/transport/webui/bot_test.go +++ b/internal/transport/webui/bot_test.go @@ -51,6 +51,66 @@ func TestHandleChatSuccess(t *testing.T) { } } +func TestHandleChatWithFileIDs(t *testing.T) { + b := newTestBot(t, 1024*1024) + b.chatHandler = func(_ context.Context, msg IncomingMessage) (string, error) { + if msg.ChatID != "s1" || msg.UserID != "u1" || msg.Text != "hello" { + t.Fatalf("unexpected message: %+v", msg) + } + if len(msg.FileIDs) != 2 || msg.FileIDs[0] != "file_a" || msg.FileIDs[1] != "file_b" { + t.Fatalf("unexpected file ids: %+v", msg.FileIDs) + } + return "ok", nil + } + + body := strings.NewReader(`{"text":"hello","session_id":"s1","user_id":"u1","file_ids":["file_a","file_b"]}`) + req := httptest.NewRequest(http.MethodPost, "/api/chat", body) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + + b.handleChat(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String()) + } +} + +func TestHandleChatWithFileIDsAliases(t *testing.T) { + tests := []struct { + name string + body string + }{ + {name: "camel array", body: `{"text":"hello","sessionId":"s1","userId":"u1","fileIds":["file_a","file_b"]}`}, + {name: "flat array", body: `{"text":"hello","session_id":"s1","user_id":"u1","fileids":["file_a","file_b"]}`}, + {name: "single key", body: `{"text":"hello","session_id":"s1","user_id":"u1","file_id":"file_a"}`}, + {name: "csv string", body: `{"text":"hello","session_id":"s1","user_id":"u1","file_ids":"file_a, file_b"}`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := newTestBot(t, 1024*1024) + b.chatHandler = func(_ context.Context, msg IncomingMessage) (string, error) { + if msg.ChatID != "s1" || msg.UserID != "u1" || msg.Text != "hello" { + t.Fatalf("unexpected message: %+v", msg) + } + if len(msg.FileIDs) == 0 { + t.Fatalf("expected file ids from alias payload, got empty") + } + return "ok", nil + } + + body := strings.NewReader(tt.body) + req := httptest.NewRequest(http.MethodPost, "/api/chat", body) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + + b.handleChat(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String()) + } + }) + } +} + func TestHandleChatMissingText(t *testing.T) { b := newTestBot(t, 1024*1024) b.chatHandler = func(_ context.Context, _ IncomingMessage) (string, error) { return "", nil }