diff --git a/.vscode/launch.json b/.vscode/launch.json index c995be1..c95919f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -6,6 +6,7 @@ "type": "go", "request": "launch", "mode": "auto", + "preLaunchTask": "Kill Stale LaodingBot Debug Processes", "program": "${workspaceFolder}/cmd/bot", "cwd": "${workspaceFolder}", "envFile": "${workspaceFolder}/configs/env" @@ -15,6 +16,7 @@ "type": "go", "request": "launch", "mode": "auto", + "preLaunchTask": "Kill Stale LaodingBot Debug Processes", "program": "${workspaceFolder}/cmd/bot", "cwd": "${workspaceFolder}", "envFile": "${workspaceFolder}/configs/env", @@ -27,6 +29,7 @@ "type": "go", "request": "launch", "mode": "auto", + "preLaunchTask": "Kill Stale LaodingBot Debug Processes", "program": "${workspaceFolder}/cmd/bot", "cwd": "${workspaceFolder}", "envFile": "${workspaceFolder}/configs/env", diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..485c2e9 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,11 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Kill Stale LaodingBot Debug Processes", + "type": "shell", + "command": "pkill -f '/LaodingBot/cmd/bot/__debug_bin|dlv.*LaodingBot/cmd/bot|dlv dap' || true", + "problemMatcher": [] + } + ] +} diff --git a/README.md b/README.md index 3842450..14af5ca 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Go-based personal Telegram Agent with: - Telegram polling transport - OpenAI-compatible LLM client - SQLite conversation memory + simple compression -- Tool registry with built-in `file` and `shell` tools +- Tool registry with built-in `file`, `shell`, and `git` tools - Default-deny security policy via allowlists - Soul markdown loading for bot personality - Skills markdown loading for capability context @@ -50,7 +50,7 @@ go run ./cmd/bot 1. Receive message and load recent memory context 2. Match relevant skill(s) from `skills/` 3. If no skill matched, respond via direct LLM - 4. If skill matched, run ReAct and call tools (`shell` / `file`) only when needed + 4. If skill matched, run ReAct and call tools (`shell` / `file` / `git`) only when needed 5. Return final answer - No `/tool ...` command is required for normal use. @@ -69,6 +69,7 @@ go run ./cmd/bot - `shell` only allows commands listed in `ALLOWED_COMMANDS`. - `file` only allows paths inside `ALLOWED_DIRS`. +- `git` only allows common git subcommands and runs inside `WORK_DIR`. - Working directory for shell is limited by `WORK_DIR`. ## Next Iteration diff --git a/cmd/bot/main.go b/cmd/bot/main.go index 5757967..a905a4c 100644 --- a/cmd/bot/main.go +++ b/cmd/bot/main.go @@ -177,6 +177,24 @@ func runMessageChannel(ctx context.Context, cfg config.Config, engine *agent.Orc } lg.Infof("starting feishu transport") return fs.Run(ctx, func(ctx context.Context, msg feishu.IncomingMessage) (string, error) { + if msg.MsgType == "file" && len(msg.FileBytes) > 0 { + content := msg.FileBytes + if msg.FilePath != "" { + if b, err := os.ReadFile(msg.FilePath); err == nil && len(b) > 0 { + content = b + } else if lg != nil { + lg.Warnf("read local file failed path=%s err=%v; fallback to in-memory bytes", msg.FilePath, err) + } + } + files := []llm.InputFile{{ + FileName: msg.FileName, + MimeType: msg.FileMime, + Content: content, + }} + // Feishu file event and user question are split into separate messages. + // Use empty text so file IDs are cached and consumed by the next text query. + return engine.HandleMessageWithFiles(ctx, msg.ChatID, msg.UserID, "", files) + } return engine.HandleMessage(ctx, msg.ChatID, msg.UserID, msg.Text) }) default: diff --git a/files/1772978682808419264_EIP_RequirementsSpecificationGLA__V2-5.pdf b/files/1772978682808419264_EIP_RequirementsSpecificationGLA__V2-5.pdf new file mode 100644 index 0000000..a5f6f50 Binary files /dev/null and b/files/1772978682808419264_EIP_RequirementsSpecificationGLA__V2-5.pdf differ diff --git a/files/1772979029589132982_EIP_RequirementsSpecificationGLA__V2-5.pdf b/files/1772979029589132982_EIP_RequirementsSpecificationGLA__V2-5.pdf new file mode 100644 index 0000000..a5f6f50 Binary files /dev/null and b/files/1772979029589132982_EIP_RequirementsSpecificationGLA__V2-5.pdf differ diff --git a/files/1772979118236039688_EIP_RequirementsSpecificationGLA__V2-5.pdf b/files/1772979118236039688_EIP_RequirementsSpecificationGLA__V2-5.pdf new file mode 100644 index 0000000..a5f6f50 Binary files /dev/null and b/files/1772979118236039688_EIP_RequirementsSpecificationGLA__V2-5.pdf differ diff --git a/files/1772979372720489345_EIP_RequirementsSpecificationGLA__V2-5.pdf b/files/1772979372720489345_EIP_RequirementsSpecificationGLA__V2-5.pdf new file mode 100644 index 0000000..a5f6f50 Binary files /dev/null and b/files/1772979372720489345_EIP_RequirementsSpecificationGLA__V2-5.pdf differ diff --git a/files/1772979941413990286_EIP_RequirementsSpecificationGLA__V2-5.pdf b/files/1772979941413990286_EIP_RequirementsSpecificationGLA__V2-5.pdf new file mode 100644 index 0000000..a5f6f50 Binary files /dev/null and b/files/1772979941413990286_EIP_RequirementsSpecificationGLA__V2-5.pdf differ diff --git a/files/1772980103865061914_EIP_RequirementsSpecificationGLA__V2-5.pdf b/files/1772980103865061914_EIP_RequirementsSpecificationGLA__V2-5.pdf new file mode 100644 index 0000000..a5f6f50 Binary files /dev/null and b/files/1772980103865061914_EIP_RequirementsSpecificationGLA__V2-5.pdf differ diff --git a/files/EIP_RequirementsSpecificationGLA__V2-5.txt b/files/EIP_RequirementsSpecificationGLA__V2-5.txt new file mode 100644 index 0000000..80f9fc2 --- /dev/null +++ b/files/EIP_RequirementsSpecificationGLA__V2-5.txt @@ -0,0 +1,2455 @@ +Software Requirements Specification for Urban Platforms (EIP Project) Page i + + + + + Requirements Specification + For + + Urban Platforms + Version 2.2 + (Consultation Stage) + + + + + Prepared by Demand-Side Engagement Stream + + + + +European Innovation Partnership for Smart Cities & Communities (EIP_SCC) +Integrated infrastructure Action Cluster – Urban Platform + Software Requirements Specification for Urban Platforms (EIP Project) Page ii + + + + +Requirements Specification Life-Cycle +Revision history + + Author Date Reason for change Phase Revision + + Dr. Larissa Romualdo-Suzuki 4/12/2015 - Initial Draft Version 0 + Outline + + Dr. Larissa Romualdo-Suzuki 21/12/2015 Working update Core Demand- Version 1 + Side Team + Consultation + + Dr. Larissa Romualdo-Suzuki 05/01/2016 Incorporate cities Development Version 2.1 + comments + + Dr. Larissa Romualdo-Suzuki 29/01/2016 Incorporate suggestions Development Version 2.2 + from Urban Platforms + Workshop held in Brussels + on the 19/01/16. + + + + +Requirements Specification sign-off: + + Name: Andrew Collinge + Role: Demand Side Engagement Lead Date + Organisation: Greater London Authority 11 January 2016 + + + + +Acknowledgement: + +This document has been developed in consultation with a representative group of European cities from +different locations, size, and level of development. Cities include: London, Amsterdam, Barcelona, +Syracuse, Berlin, Gent, Valencia, Murcia, Derry, Copenhagen, Scottish cities, Porto, Riga. We wish to +recognize the commitment, support and ongoing contribution from these cities to develop a common +statement of needs. + Software Requirements Specification for Urban Platforms (EIP Project) Page iii + + + + +Table of Contents +Table of Contents ...................................................................................................................... iii +1. Introduction ............................................................................................................................1 + 1.1 Purpose and Scope of this Document.................................................................................. 1 + 1.2 Context ..................................................................................................................................... 1 + 1.3 Intended Audience .................................................................................................................. 2 + 1.4 Definitions, Standards, and Framework ............................................................................... 3 + 1.5 How to Use this Document .................................................................................................... 2 + 1.6 Product Scope and Perspective ............................................................................................ 5 + 1.7 The Urban Platform Development Stack ............................................................................. 7 + 1.8 User Classes, Characteristics, and User Access ............................................................... 8 + 1.9 User Documentation ............................................................................................................. 10 + 1.10 Design and Implementation Constraints ............................................................................ 10 + 1.11 Assumptions, Alignment with other Action Clusters and Policies ................................... 10 +2. Urban Platform Value Proposition, Use Cases and Functional Requirements ..12 + 2.1 From Value Proposition to Platform Specifications .......................................................... 12 + 2.2 Overall Goal: “City data is exploited to its full potential” .................................................. 13 + 2.3 SUB-GOAL 1: City data is collected in an intelligent manner ......................................... 15 + 2.4 SUB-GOAL 2: City data is managed in a safe and intelligent manner .......................... 20 + 2.5 SUB-GOAL 3: City data is orchestrated in a marketplace ............................................... 23 + 2.6 SUB-GOAL 4: City data is offered in an accessible manner ........................................... 28 + 2.7 SUB-GOAL 5: User’s experience is enhanced by the provision of value-added + services............................................................................................................................................... 31 +3. Non-functional Requirements..........................................................................................34 + 3.1 Run-time Quality Requirements .......................................................................................... 34 + 3.2 Non Run-time Quality Requirements.................................................................................. 37 + 3.3 List of Non-Functional Requirements ................................................................................. 38 +4. Other Requirements ...........................................................................................................39 + 4.1 Minimal Descriptive Metadata Required from Data Provider .......................................... 39 + 4.2 Urban Platform Supported Data Formats .......................................................................... 40 +5. Conclusion & Forward Plans ...........................................................................................40 +Appendix ......................................................................................... Error! Bookmark not defined. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 1 + + + + +1. Introduction +1.1 Purpose and Scope of this Document +The purpose of this document is to provide a common core set of city-needs-led requirements, co- +developed and validated by EU cities, to support the acquisition of “Urban Platforms” by EU cities. +The requirements are made openly available. The ambition is that adoption of these requirements +by EU cities will lead to reduced pre-procurement times, increased confidence in platform designs, +greater levels of collaboration (particularly amongst smaller cities), innovation in business models, +more affordable solutions, and a more secure basis for industry to apply its innovations. This will +lead to accelerated adoption of urban platforms by EU cities, so that they can exploit the potential +of the growing volumes of city data, and improve the services to and outcomes for their residents, +business community and visitors. + +This document provides a primary input to the EIP_SCC Urban Platform Supply-Side and +Standardisation workstreams as a reference for the development of further technical and +operational materials. The requirements illustrate the purpose and complete declaration for the +development of system, as well as system constraints, interface and interactions with other +external applications. + +The scope of this document is limited to common services data platform requirements, and issues +related to the management of city data within an Urban Platform. This considers the full life-cycle +of city data, including: the maintenance of the inventory of existing city data; development of +functional requirements for common services data platform from a city-needs perspective; +identification of metadata and format standards for city data; and identification of policy issues +related to the data platform. It includes in the scope the functionality of software and systems used +to handle existing data catalogues (e.g., open data portals). + +Commercial off-the-shelf (COTS) software or hardware, as well as databases, which manage data +that are not part of Urban Platform’s collecting responsibility (e.g., mobile applications, proprietary +Internet of Things, proprietary data catalogues) are outside the scope of these requirements. + + +1.2 Context +The initial EIP_SCC Demand-Side survey1 clearly indicated that the vast majority of EU cities did +not have an urban platform; that there were a number of significant barriers to adoption (including +funding, capabilities, and the willingness to work across service ‘silos’); that there was limited +collaboration in this area; and as a result most cities were ‘sitting on the fence’. The opportunity +clearly is to unblock this. Notably because urban platforms provide a vital foundation for smart city +infrastructure and service improvements. So their adoption is considered a strategic priority by the +EIP_SCC, and by a growing number of cities. + +There are a number of clear trends that any city, and thus these requirements, must recognise: + • The astounding increase in volumes of city data, driven notably by IoT/sensor/M2M + implementations, and social media + • The pressure to improve and make open data from public (and private) sources; + +1 + Survey performed Q1’15 and reported on the EIP_SCC Marketplace site + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 2 + + + • The continued reality of austerity that drives cities towards transformative solutions + +Cities presently typically have a mix and variety of qualities of legacy systems, with various +degrees of technical ‘platform or platforms’ in place. It is recognised that the roadmap a city might +follow to transition to a more aligned and functional state will differ in many ways; and that the end +point does not envisage ‘one (physical) platform’. However cities will benefit from adopting more +consistent conceptual, and logical architectures, which will lead to more efficient and effective +physical operations, and greater benefits in terms of exploiting city data. + +Furthermore it must be recognised that cities will continue to wrestle with capacity challenges to +improve quality, availability, governance, and appropriate monetisation of city data. + + +1.3 Intended Audience +This document is co-developed by and initially intended for the growing number of city members of +the Demand-Side Engagement Stream of the EIP_SCC Urban Platform initiative. Its formal release +as a common statement of requirements by representative EU cities will form the basis for +commitment by EU cities to the initiative by way of a Letter of Intent (LoI). The spirit of which seeks +to ensure broad use of these requirements by EU cities (and recognition of deviations) to help +open the smart city market. + +It is also intended to be used by Industry in setting an agreed city-needs-led set of requirements +which can become more consistent and common across EU cities. This will lead to advantage for +cities and Industry. + + +1.4 How to Use this Document +Cities are encouraged to use this document as a guiding reference for the development of their +own platform specifications. + +The following steps offer a logical means to extract the full value of this document: + + + 1. Identify the priority service outcomes that the city seeks to achieve to steer the overall + prioritisation of urban platform development + 2. Perform a city data mapping exercise to develop a picture of the data landscape, including: + sources, volume, variety, temporal factors and sensitivity, data licensing and ownership + policies. + 3. Map out existing ICT system resources across the city in order to identify those resources + with the greatest potential for reuse, identify gaps and provide the foundation for a data + strategy and technology plan to fill them. + 4. Identify the requirements within this document that best suit the city needs - alongside any + other emerging requirements – and refine and prioritise them in accordance to their data + strategy defined in step 1 and 2. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 3 + + + 5. Cities can create their own specification document in their own language and refer to the + requirements of this document by using the unique identifier of the requirements (which + signals to providers the similarities and deviations from the core set) + 6. Determine what ICT infrastructure and sets of requirements are needed citywide to support + the urban platform. + + + + +1.5 Definitions, Standards, and Framework +To ensure a robust basis for market adoption we have set in place a number of foundations. Firstly, +a common definition of terms. Secondly, the adoption of some core accepted standards. Thirdly, +an underlying model to develop requirements. + +1.5.1 Working Definitions +For the purpose of this document we assume the working definitions developed within and shared +across all working groups of the Urban Platform initiative + +• An ‘Urban Platform’… + + o Implements a logical architecture/content/design that brings together (integrates) data flows + within and across city systems and exploits modern technologies (sensors, cloud services, + mobile devices, analytics, social media etc); + o Provides the building blocks that enable cities to rapidly shift from fragmented operations to + include predictive effective operations, and novel ways of engaging and serving city + stakeholders; + o Transforms, in a way that is tangible and measurable, outcomes at local level (e.g. increase + energy efficiency, reduce traffic congestion and emissions, create (digital) innovation + ecosystems, efficient city operations for administrations and services). + +• ‘City Data’ is… + + o Data that is held by any organisation - government, public sector, private sector or not-for- + profit - which is providing a service or utility, or is occupying part of the city in a way that can + be said to have a bearing on local populations and the functioning of that place; + o Consists of varied characteristics such as static, near-real time or in the future, real time, + descriptive or operational. + o It will be to a greater extent generated by individual citizens and this too (with due + consideration to privacy and a strong trust framework) can be considered city data. + + +For the purpose of this report, the following terms and definitions4 apply. + + o Open data: non-privacy-restricted and non-confidential data. Produced with either public or + private resource and is made available without any restrictions on its usage or distribution. + o Private data: restricted and/or licensed data including permission, charging, privacy, + publication and distribution. Produced with either public or private resource. + o Commercial data: restricted and/or licensed data including permission, charging, privacy, + publication and distribution. Produced with either public or private resource. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 4 + + + o Sensory data: open and/or restricted data collected by different sensors, actuators and + devices owned by public and private sector, and citizens. Sensory data is usually diverse in + nature and architectural features, mostly location and time dependent, and present different + levels of quality. + +These types of city data are produced within the public and private sectors and crowdsourcing +initiatives, and have their respective suppliers and distribution strategies (e.g. reports, APIs, +datasets). + + o Public sector data: restricted data relating to location, national security, commercial + sensitivity and privacy. Data produced, collected or funded by the public sector. + o Private sector data: data produced, collected or funded by the private sector, which can be + open, private and commercial data. In case of private data, the restrictions of usage and + distribution are decided by the individual businesses. + o Crowd-sourced data: data provided, collected and distributed by humans through the use + of digital technologies and social media. + + +1.5.2 Use of Standards + +• We seek to ensure Urban Platforms in EU cities are designed with regard to international best + practices for data repositories. +• As new guidance and standards are developed the three streams of the EIP_SCC2 Urban + Platform initiative will review them for broader use. (Note: the SCC03 EC project and + consortium is explicitly linked within this initiative and will act on Urban Platform standards). +• For now, this document assumes a modified IEEE 830-19983 layout for software requirements + specification, as the basis to capture a high-level statement of the urban platform’s + requirements. + + +1.5.3 The Organising Framework + +This document uses the dynamic business models framework4 illustrated in Figure 1 to organise +and guide the definition of goals, the development of use cases, and definition of requirements +during the life cycle of urban platforms. Future iterations of this dynamic design process are +Procurement, Roll-Out and Market phases. Consequently, this document may include, alter, +prioritize, extend or remove specifications as they become known. + +The principal goal addressed is that of “exploiting the value of city data through urban +platforms”. From this we have developed five common sub-goals. Each includes: a description; +rationale; drivers; and actions. These sub-goals are elaborated through a total of 14 use cases and +developed into a total of 116 functional and non-functional requirements as illustrated in Figure 2. +The use of the requirements by Industry will be subject to a further update. + + + + +2 + EIP Integrated infrastructure Urban Platform Governance Model, Version 3, 28th July 2015. +3 + IEEE 830-1998 Standard for Software Requirements Specifications, 1998. +4 + Framework for Data Infrastructures Design from “Data as Infrastructure for Smart Cities”, Suzuki-LCSR, +PhD Thesis, UCL, 2015. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 5 + + + + + Figure 1. Organising Framework4. + + + + + Sub-Goal 1: City data is provided + Use Cases: 3 Requirements: 34 + in a harmonised way + value of city data through an + Principle Goal: Exploit the + + + + + Sub-Goal 2: City data is managed + Use Cases: 3 Requirements: 36 + in a safe and intelligent manner + urban platform + + + + + Sub-Goal 3: City data is + Use Cases: 2 Requirements: 12 + orchestrated in a market place + + + + + Sub-Goal 4: City data is offered in + Use Cases: 3 Requirements: 11 + an accessible manner + + + + + Sub-Goal 5: User’s experience is + enhanced by the provision of Use Cases: 3 Requirements: 8 + value-added services + + + + Figure 2 Structure of this document. + + + + +1.6 Product Scope and Perspective +The EIP Project’s urban platform is an open common architecture which serves for city data +collection, management and distribution. An urban platform is intended to support the widespread +exploitation of city data by humans and machines in the urban environment. Figure 3 illustrates a +holistic high level overview of the urban platform the EIP project intends to deliver. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 6 + + + +The reference architecture for urban platforms should: + +• Cater for interoperability between urban infrastructures +• Enable replicability of the solutions/platforms city to city +• Scale without technical constraints and excessive cost increase +• Provide open APIs and SDKs +• Enable Real Time capabilities +• Support implementation of functional and technical capabilities + + + + + Figure 3. High level overview of the urban platform (currently approved EC DG CNECT). + + +The current urban platform market is nascent. Many software vendors offer such a platform, though +many requirements and expectations of the stakeholders of city data are not (fully) addressed. As +a result, current urban platforms are often more costly to design and maintain, less reusable and +often not interoperable platform-to-platform, and susceptible to information fragmentation and +overload. + +The urban platform which the EIP initiative intends to design takes a step beyond the platforms +currently on the market by ensuring the requirements are fully founded on a co-created and +common set of representative city needs, from which it solicits suitable industry input, and an open +and managed collaboration between industry, cities and communities, and others, in order to take +into account their needs and concerns. To do this, it is necessary to take a technology agnostic +approach to design an open and common reference architecture for urban platforms. This platform +must ensure data is collected and sustained in accordance with well-stablished standards, +managed in a robust manner so that it can handle high level supply and demand of data, and +distributed across different value chains, systems and stakeholders. The ability to handle high level +of city data supply and demand while being user secure and accessible enough for city-wide +exploitation of data is one of many keys to the success of urban platforms. This is central to the +design and implementation of urban platforms. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 7 + + +1.7 The Urban Platform Development Stack +The requirements specification of this document is based on the Urban Platform Development +Stack illustrated in Figure 4. The stack is composed by five domains (represented as layers in the +stack) necessary to fully implement an urban platform software suite as shown in Table 1. Each +domain comprehends a set of requirements necessary to the design of a common and open urban +data service platform. The elicited requirements are used to define a technical architecture which is +simple enough to be comprehensible at least at a high level of abstraction. The platform should be +conceptually decomposable into its major subsystems, the platform’s functionality reused by many +services and external applications should be identifiable, and interactions between the platform +and services, data providers and data consumers should be well defined and explicit. + +The first layer of the stack “Societal needs” concerns to outcomes we strive for within a portfolio of +city service domains. An urban platform should recognise societal needs and wants as the starting +point for city data service offering. Ultimately, an urban platform aims to provide tailor made and +compelling engaging services for the users. The Services and Business models layers concerns +with delivering data services which carefully targets the needs and expectations of the different +users of the urban platform, and explore use cases and commercial models where data is used to +deliver different forms of value. The city data layer concerns with the mechanisms necessary to +transform urban platforms into a foundation for widespread exploitation of data, including handling +data architectural features, data usability, semantics and quality aspects. The urban platform layer +concerns to the technology foundation to configure, share, and interpret exponentially increasing +volumes city data and services. Finally, the Infrastructure layer concerns with the base level +connectivity that supports the platform to be scalable and reliable in the long run. + + + + STACK OUTPUT + + Requirements to deliver new digital services that will address the + Societal Needs societal needs of cities in a positive manner that relates to political + narratives. + + + Services & Requirements to new profitable business models and the development + Business Models of an increase range of new and engaging services in the smart cities. + + + + City Data Requirements to provide all city data stakeholders ready access and + delivery of all city data that unpins the decision making process in smart + cities. + + + Urban Platform Requirements to put in place applications together to build a foundation + for the widespread exploitation of data. + + + Infrastructure Requirements to deliver the backbone infrastructure that will be used to + capture the opportunities of digital technology and data to enable + transformation. + + + Figure 4. Urban Platform Development Stack. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 8 + + + Table 1. Urban Platform Development Stack + Layer Rationale + Societal Needs - Accessible services and data necessary to solve social problems + and drive innovation; + - Parameters that influence user’s experience while interacting with + services (e.g. usability, feeling of security and trust); + Services and Business - Tailor-made data services which careful targets the needs of users + Models and businesses; + - New potential and cost-effective beneficial services that could be + rolled out across cities of different sizes; + - Use cases where data is used to deliver different forms of value. + City Data - Data architectural features (e.g. volume, variety, temporal factors + and sensitivity); + - Data licensing, policies and regulations to exploit data to full effect; + - Minimum metadata requirements; + - Data usability and reusability aspects of humans and machines. + Urban Platform - Holistic and interoperable solutions; + - Integrated approaches which ensures that services fit together and + that synergies can be exploited; + - Data management mechanisms to ensure data integrity and + compliance with data protection regulations + - Extension capabilities to accommodate additional functionality at + later stage at a fair and transparent cost. + + + +1.8 User Classes, Characteristics, and User Access +The users of the Urban Platform include end-users, such as the general Public, public and private +organisations; data providers; service providers; and the platform providers who will be +working with the providers of city data and services, and managing the content, defining policies +and regulations of the platform. A crucial feature of an urban platform is the provision of the +various access levels required by the different types of users. Particular uses need different +access levels to some data than the general public. Data publishers will require access to the +Urban Platform in order to ingest, administer, manage, preserve and access their resources. This +will require multiple levels of access to city data and its respective metadata. Table 2 provides a +description of each class of users. + + + Table 2. Actors + User Class Rationale + Platform - Maintains the ecosystem of data, services and users; + Provider - Defines standards, licenses and regulations and provides terms and conditions + for platform usage and the commercial exploitation of data and services; + - Decides who are allowed to join the value network of data and services + providers; + City Data - Publishes open and proprietary data into the platform; + Publisher - Manages and maintain resources in the platform accordingly to terms and + conditions. + Data - Deploys open and commercial data services into the platform (e.g. data + Services visualisation, data cleansing, data integration tools); + Provider - Manages and maintain resources in the platform accordingly to terms and + conditions. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 9 + + + City Data - Consumes open and proprietary data provided in the platform; + Consumer - Uses open and commercial data services provided in the platform; + - Provides feedback on data and services provision; + + +1.8.1 End-User Access + +City data consumers will need to access and use the city data residing in the Urban Platform. End- +users will be able to search metadata and full text within datasets (when available), and obtain city +data in open formats readily available to both humans and machines such as CSV, XML, JSON. +Some end-users may require different access rights to city data. The 2 major end-user groups that +have been identified are: + +• Open data users, including both national and international users (humans and machines). + Open access to some city data may be restricted by licensing terms (e.g. commercial data), + embargo periods, copyright, etc. + +• Private data users, which need to use the Urban Platform to obtain commercial city data. Data + access is available via data subscriptions or when purchase requirements and licenses are + waived by the data provider. + + +1.8.2 City Data Publisher Access + +A broad data provider level access is needed for stakeholders (humans and machines) working +with the urban platform and their respective data in it. Basically, data publishers will carry out the +following activities: +• Data publication access, available to publishers adding new data and metadata, checking the + quality of datasets, manipulating data, performing format conversions, defining data-access + level, tariff for consumption when applicable, and licences. + +• Data maintenance access, for publishers reviewing or editing appropriate data and metadata in + the urban platform. Data publishers can view data and add to or edit metadata without + changing the data itself. They should be provided with access to feedback from users to + investigate problem in their resources (e.g. missing data, inconsistent metadata), and statistical + information about how their resources are used by users. + + +1.8.3 Data Services Provider Access + +This is the second most restrictive access level providing rights to deploy services in the platform. +Basically, data service providers will carry out the following activities: + +• Data services deployment access, available to service providers adding new mechanisms or + integrating new applications, testing and validating integration, defining data-access level and + tariff for service usage. + +• Data services maintenance access, for services providers reviewing, extending or editing + applications in the urban platform. Data services providers can view their services deployed + and add to or edit access level and tariff without having to deploy the services again. They + should be provided with access to feedback from users to investigate problem in their services + (e.g. bugs, scalability issues), and statistical information about how their services are used by + users. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 10 + + +1.8.4 Platform Provider Access + +This is the most restrictive access level providing ultimate rights to the system and is required for +its management, development, and assigning appropriate rights to data and services providers. +Policies and regulations, license agreements are also defined by the provider of the urban +platform. Platform providers should be provided with the means to follow up on civic engagement +(e.g. feedback, request for city data) and on the provision of city data and services. + + +1.9 User Documentation +• City Data Consumers: Provide license terms and conditions associated to consuming + data and services provided in the platform, documentation of APIs and guide to + discover city data in the platform. +• City Data Providers: Provide data publication documentation describing the minimum + metadata requirements, formats accepted, step-by-step guide to publish accurate city + data. +• Data Service Providers: Disclosure technical and architecture blueprint details in + order share and outsource expertise, and partnerships, and integrate supporting + partners’ solutions into the platform itself. + + +1.10 Design and Implementation Constraints +1.10.1 Design Constraints + +• Lack of standards agreement for metadata representation. +• City data found in existing data catalogues may require special consideration concerning the + type of formats and datasets that must be stored within the platform. +• Requirements mismatch due to increased number of stakeholders involved in the design + + +1.10.2 Implementation Constraints + +• Evaluation and testing of software options is expected to occur prior to selection and + implementation of a production urban platform. +• Budget costs are unknown until evaluation of software options is completed. + + +1.11 Assumptions, Alignment with other Action Clusters and Policies +1.11.1 Assumptions + +The assumptions in Table 3 have been identified by the Demand Side Engagement Stream as +relevant to this Requirements Specification. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 11 + + +Table 3. Assumptions + + # ASSUMPTION + + 1 The providers of city data and services will be responsible to maintain their resources in the platform. + + 2 All city data must meet the minimum metadata requirements and use the standards adopted by the platform. + + 3 The platform shall consider open Source as an optional commercial model, with open standards as a principle + + 4 The system design and architecture should minimize fragmentation of city data in the urban platform. + + 5 To the extent possible, automation should be used for the extraction of descriptive and technical metadata. + + 6 The platform must be designed in a way it accommodates additional functionality at later stage at a fair and transparent cost. + + The platform must be a modular based architecture which relies on stable and well-defined open interfaces to ensure + 7 + interoperability between the platform, services and the applications provided by service providers. + + The platform will offer open and well-documented API’s and clear service descriptions and contracts that is offered for reuse by + 8 another party to foster open innovation in the city, which means that developers and interested individuals openly utilize the + resources provided. + + 9 Adopt open and published European and International standards where possible. + + The platform must be flexible enough to accommodate different local, National and International data protection, licensing and +10 + commercialization regulations. + +11 Platform providers will monitor emerging technologies in order to maintain and improve the architecture. + + +12 Platform providers will monitor emerging information standards, including metadata standards and data interface standards. + + +13 Platform providers will monitor new commercial models for city data exploitation + + + + +1.11.2 Alignment with Citizen’s Focus Action Cluster +This specification document is aligned with the principles defined in the Citizen Focus5 Action +Cluster of the European Innovation Partnership on Smart Cities and Communities. Citizen Focus’ is +about “working together with citizens to realize public interests at the intersection of ICT, mobility +and energy in an urban environment”. We recognize citizens as owners of and participants in +the creation and delivery of city data and digital services, and we specify requirements to deliver +new digital services that will address the societal needs of cities in a positive manner that relates to +political narratives. Societal needs and wants are considered the starting point for city data service +offering by the urban platform. The requirements in this document were elicited considering + +• Human behaviour and needs as important as technology; +• The services and data that solve social problems and drive innovation; +• The mechanisms that make data and services more accessible to users; +• The factors that influences user’s experience while interacting with services provided (e.g. + usability, feeling of security and trust); + + +5 + https://eu-smartcities.eu/node/1333 + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 12 + + +1.11.3 Policies to be developed + +The following policies have been identified by the Demand Side Engagement Stream as relevant to +this Requirements Specification. + +• Data formatting and Metadata Schemes: Urban platforms will require more expansive, robust + and useful data encoding and conversion that what is available in existing data catalogues. + Data preservation policies should be developed to allow data to be stored in formats that can + be migrated, associated with metadata and ontologies to become both humans and machines + readable and understandable, ongoing monitoring for data obsolescence, and migrating data to + systems environments as needed to ensure their continued availability. Current Metadata + schemes (e.g. open data, sensory data ontologies) should be reviewed to see if it meets + current needs for city data management. It is possible that the needs of the urban platform + designed here may require new or additional schemas. + +• Data commercialisation: The commercial exploitation of city data and their funding models + are unexplored concepts that we are committed to address. There is an urgent need to define + license agreements and fair commercial and subscription models to allow interoperable open + and proprietary data to co-exist in the platform. + +• Data publication and services deployment: Policy development will be needed regarding + ingesting data and deploying data services into the platform, including which users/machines + will be authorized to submit data for publication, the minimum requirements for data submitted + by open and proprietary data publishers, and the removal of resources and services from the + platform. + + +2. Urban Platform Value Proposition, Use Cases and +Functional Requirements +2.1 From Value Proposition to Platform Specifications +This document uses goal-oriented modelling for eliciting, elaborating, structuring, specifying, +documenting, and modifying requirements. Goals represent the objectives which the urban +platform should achieve through cooperation of actors in the intended system and in the +environment. They capture, at different levels of abstraction, the various objectives the urban +platform under design should achieve. Through goals modelling we consider how the value +proposition and intended solutions connects across the stack, how the urban platform meets city +goals, why the system and its functionality are needed, and how the stakeholders’ interests may be +addressed. + +In our specification, we present the overall goal (the value proposition) that the urban platform +should aim to achieve in order to be considered as a viable final product, and a set of sub-goals +(intended solutions) it should maintain in the long run so that the overall goal can be unceasingly +achieved. By using this approach, the low-level technical requirements can be traced back to high- +level strategic objectives of the urban platform. The formal notations used in this document are: +Achieve [“Name of Overall Goal”] and Maintain [“Name of Sub-Goal”]. The requirements of the +Urban Platform are noted for each of the sub-goals, and are presented as a series of statements +regarding the capabilities needed in to achieve the overall goal of the Urban Platform. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 13 + + +2.2 Overall Goal: “City data is exploited to its full potential” +An urban platform is an organization of people and systems, which has accepted the responsibility +to preserve city data and make it available for all the stakeholders of smart cities. Ultimately, an +urban platform is a foundation for the full exploitation of city data. Hence, the major goal an Urban +Platform must achieve is “city data is exploited to its full potential”. To achieve this high level goal, +the urban platform must maintain in the long run the five sub-goals illustrated in Figure 3. + +Each one of the defined sub-goals co-enables the achievement of the specified high-level (overall) +goal of the urban platform. The sub-goals include the ingestion of city data, metadata generation, +data management, data storage, access, preservation, and administration, provision of engaging +services in the smart cities. These sub-goals are discussed in details in the following sections. + + + + Achieve [City data is exploited How we achieve the platform overall goal? + Why do we need the sub-goals? + to its full effect] + Goal + + + + + Maintain [User s experience is + Maintain [City data is provided + enhanced by the provision of + in a harmonised way] + S ub-Goal 1 S ub-Goal 5 value-added services] + co-ena bles + + + + + Maintain [Resources are managed in Maintain [City data is offered + a safe and intelligent manner] in an accessible manner] + S ub-Goal 2 S ub-Goal 4 + + + + + Maintain [City data is orchestrated + in a market place] + S ub-Goal 3 + + + + + Figure 3. Platform High-Level Goal and its respective sub-goals. + + + +2.2.1 Urban Platform Boundary + +The use case diagram illustrated in Figure 4 identifies the boundaries between the actors (either +automated or human) and the urban platform. We have arrived at the urban platform boundary by +inspecting each business use case and determining, in conjunction with the stakeholders needs, +which part of the business use case should be implemented and what part should be done by an +outsourced product (e.g. Billing System) using the framework 4. + +This task is technology agnostic and takes into account the abilities of the users/actors, the +constraints, the goals of the urban platform. Table 4 maps out the use cases with their respective +sub-goals and actors. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 14 + + + + + Manage Services + + S ub-Goal 5 + + + Data Service + Publish City Data + Provider + S ub-Goal 1 + + + <> Deploy Data Services + rules + S ub-Goal 5 + + Authenticate in the + Manage Resources <> + Platform Platform + S ub-Goal 1 Provider Database + <> Manage Infrastructure System + S ub-Goal 2 + + Utlise Data Services + City Data <> + Publisher S ub-Goal 5 + + Store City Data Management + <> + S ub-Goal 2 + Systems Services + System + + <> + City Data + Consumer + QoS Monitoring + Transmit Data + System + Register in the Platform S ub-Goal 2 + + + + + <> + rules + + <> + rules + Platform + Provider + rules + + Commercialise Data + Consume City Data + Services City Data + S ub-Goal 3 S ub-Goal 4 + Consumer + <> + + + Commercialise + City Data Discover City Data + S ub-Goal 3 + S ub-Goal 4 + + Billing + Management + System + + + + Authenticate in the Register in the + Platform Platform + + + + + Authenticate Authenticate + Register Consumers Register Publishers + Consumers Publishers + + + + + Figure 4. Simplistic overview of the use cases identified in the early stages of the platform design. + + + + +Table 4. Use Cases Mapping with Sub-Goals and Actors + Sub-Goals Use Cases ID Specialised Use Cases Actors +HIGH-LEVEL GOAL: City data is exploited to its + + + + + Publish City Data UC1 User publishes city data via data API’s + City Data Publisher + 1. City data is Register as a publisher + UC2 User manually uploads datasets + provided in a in the Platform + harmonised way + full effect + + + + + User manages resources + Manage Resources UC3 City Data Publisher + User tracks resources usage + + Store City Data UC4 + 2. City data is + managed in a safe Transmit Data UC5 - Management Systems + and intelligent + manner + Manage Infrastructure UC6 + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 15 + + + City Data Publisher + Set commercial city data + Platform Provider + Subscribe to proprietary data City Data Consumer + Commercialise City + UC7 + Data + Manage commercial data City Data Publisher + + 3. City data is Manage data subscription City Data Consumer + orchestrated in a + Data Services Provider + market place Set commercial data services + Platform Provider + Subscribe to commercial services Data Services Consumer + Commercialise Data + UC8 + Services + Manage commercial services Data Services Provider + + Manage services subscription Data Services Consumer + UC9 + Register in the Platform City Data Consumer + - + 4. City data is Discover City Data City Data Consumer + UC10 + offered in an + User consumes city data via data + accessible manner + API’s + Consume City Data UC11 City Data Consumer + User downloads datasets + 5. User’s Data Services Provider + Deploy Data Services UC12 + experience is Platform Provider + enhanced by the + Manage Services UC13 - Data Services Provider + provision of value- + added services + Utilise Data Services UC14 City Data Consumer + + + + + 2.3 SUB-GOAL 1: City data is collected in an intelligent manner + Description: The urban platform enables the owners of city data to easily publish both historic and + data streams in the platform, as well as their associated metadata. + Rationale: This sub-goal is maintained by the services and functions to accept the publication of + city data from data providers (of both open and proprietary data) and prepare the contents for + storage and management within the urban platform. Functions include receiving data, performing + quality assurance on data, verifying data formatting and document standards, associating meta- + data information, and coordinating updates to databases and resources management. + + Achieve [City data is exploited + to its full effect] + Goal + co-ena bles + + + + + How we achieve the platform overall goal? + + + + Maintain [City data is provided in + a harmonised manner] + S ub-Goal 1 + + + + +What actions can maintain the sub-goal? + + + + Achieve [Register Publisher] Achieve [Publish City Data] Achieve [Manage Resources] + UC1 UC2 UC3 + + + + + Figure 5. Sub-Goal 1 “City data is collected in an intelligent manner” refinement. + + + Drivers: Ensure data is published in an easy and uniform way. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 16 + + +Actions: For Sub-Goal 1 to be maintained in the long-run it requires the efficient realisation of use +cases: “Publish City Data” and “Manage Resources”, as shown in Figure 5. + + +2.3.1 Use Case: Register Publisher +ID: UC1 +Refines: SUB-GOAL 1: City data is collected in an intelligent manner +Pre-condition: Data Publisher is not logged in the system +Actors: Data Publishers +Rationale: Data Publishers can register in the platform and request approval to submit city data. +They provide valid registration details (to be defined) and wait for registration confirmation. +Platform Providers may authorise or not data publishers to offer both open and proprietary city data +in the platform. Data submission agreement is a formal agreement between the Data Provider and +the Urban Platform defining the terms of the content, standards, metadata creation, and license +agreement. The Urban Platform will proactively work with Data Providers to agree on the content, +quality and format of city data. Agreements between Platform and Data Providers may be +renegotiated on a periodic or ad-hoc basis. +Refines into requirements: FREQ.1 to FREQ.5. + + Use Case Basic Stimulus and Responses + 1. The platform prompts the user for a username and password or + register new account. + 2. The user selects registration option. + 3. The platform prompts user for publisher registration information (e.g. + username, password, organisation) + 4. The user enters in their information. + UC1. Register 5. Platform verifies information and creates account. + Publisher o If non-valid information, platform shows error message and + returns to step 1. + 6. Platform provider is requested to approve the account + o Platform acknowledges registration has been successful + o If non approved, platform shows error message and returns to + step 1. + 7. End of registration + + +2.3.2 Use Case: Publish City Data +ID: UC2 +Refines: SUB-GOAL 1 - City data is collected in an intelligent manner +Pre-condition: User is authenticated in the platform +Actors: City data publisher +Rationale: The Publish City Data function provides the appropriate mechanisms to receive city data from +authorized data providers. Data may be manually uploaded or submitted via APIs. In general, data providers +with whom the Urban Platform negotiates submission agreements are the providers of proprietary city data +(those producing published material, i.e. publishers) and open data, and they can be either humans or +machines. The providers of the Urban Platform will provide data providers with specifications on the content, +quality and format of data, and publication terms and conditions. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 17 + + +The Publish City Data function may represent a legal transfer of custody for the data in the urban platform, +and may require that special access controls be placed on the contents. This function provides a +confirmation of receipt of data publication to the Producer, which may include a request to resubmit data in +the case of errors resulting from the submission. +Once data has arrived, it must undergo several reviews, including virus checking, format compliance, +metadata minimum requirement agreement, quality and anticipated content and data formatting. The +platform must include the ability to record all actions and decisions made concerning the publication of city +data. The reasons for publication failure (e.g. missing metadata information, non-valid dataset) will be +provided back to the city data publisher. In some cases, the provider can then resubmit corrected data and +metadata information, while in other instances data publication refusal criteria should prevent the publisher +from submitting the same dataset at a later time period (e.g. in cases of suspicious datasets – copyrights +violation, viruses). When data is successfully submitted (either via APIs or manual upload), it will be +processed/prepared for storage into the platform’s database. +Specialised Use Cases: The Use Case Publish City Data data is distinguished into two +specialised Use Cases: “User publishes city data via data API’s (UC2.1)” and “User manually +uploads datasets (UC2.2)”. +Subordinated Use Cases: “Store City Data (UC4)” +Refines into requirements: FREQ 6 to FREQ.26. + + + Specialised Use Cases Basic Stimulus and Responses + 1. Platform provides user with an interface for static data + publication + 2. User selects datasets to be uploaded + 3. User provides metadata associated with the data (license, + provenance, ownership, semantics) in accordance with defined + standards + UC2.1. User manually 4. User requests data publication + uploads datasets 5. Platform quickly process user’s request for data publication + 6. Platform validates data submitted + o If valid data, platform acknowledges data publication has + been successful + o If non-valid data, platform shows error message and returns + to step 1. + 7. End of data publication + 1. Platform provides user with an interface for real-time data + publication + 2. User input data API information + 3. User provides metadata associated with the data (license, + provenance, ownership, semantics) in accordance with defined + standards + UC2.2. User publishes 4. User confirm information and request data publication + city data via data API’s 5. Platform quickly process user’s request for data publication + 6. Platform validates data submitted + o If valid data, platform acknowledges data publication has + been successful + o If non-valid data, platform shows error message and returns + to step 1 + End of data publication + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 18 + + +2.3.3 Use Case: Manage Resources +ID: UC3 +Refines: GOAL 1: City data is collected in an intelligent manner +Pre-condition: User successfully authenticates in the platform +Actors: City data publisher +Rationale: Manage resources provides the services and functions for updating, maintaining and +accessing both data and metadata, as well as tracking the usage of resources by users. Ideally +the owners of the resources should be the only authorised user to manage resources, and other +authorised users can track the usage of the resources in the platform. The platform must provide a +database update response indicating the status of the update, avoid update errors to be +propagated in the platform, and should keep an audit trail of all actions to enable rollback. Data +usage tracking includes performing queries on the data management data to generate result sets, +and producing reports from these result sets. +Specialised Use Cases: The Use Case Manage Resources data is distinguished into two +specialised Use Cases: “User manages resources (UC3.1)” and “User tracks resources usage +(UC3.2)”. +Subordinated Use Cases: “Transmit Data (UC5)” +Refines into requirements: FREQ.27 to FREQ.25. + + + Specialised Use Cases Basic Interactions and Responses + 1. Platform provides user with an interface for resources + management (e.g. data and metadata, data usage) + 2. User chooses to edit or delete data + 3. If edit, user revise metadata associated with the data (license, + provenance, ownership, access-control, semantics); + UC3.1. User manages 4. If delete, user selects dataset(s) to be removed + resources 5. User confirms action + 6. Platform quickly process user’s request + 7. Platform confirms execution of request + o If valid request, platform acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 8. End of resources management + 1. Platform provides user with an interface for resources + management (e.g. data and metadata, data usage) + 2. User chooses to visualise usage information of a dataset + UC3.2. User tracks 3. Platform quickly process user’s request for data usage + resources usage information + 4. Platform provides user with statistical information about data + usage and data users anonymised information + 5. End of data usage tracking. + + +2.3.4 Functional Requirements + + Req. ID UC. ID Description Priority Domain + Societal Needs, + FREQ.1 UC1 Allow data publishers to register to submit data for publication Must + Platform + Tracks data publication agreements between Data and Platform Business Needs, + FREQ.2 UC1 Must + Providers Platform + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 19 + + + Store terms of agreements, and use them to monitor/review/process + FREQ.3 UC1 Must City Data, Platform + data submissions. + Able to add and edit terms of agreement, based on access of level of Business Needs, + FREQ.4 UC1 Must + user. Platform + + FREQ.5 UC1 Data publications are managed and monitored Must City Data, Platform + + Allow authenticated users from across different organisations to City Data, Platform, + FREQ.6 UC2 Must + publish city data Business Needs + Provide authorization mechanisms for users and sensors to publish + FREQ.7 UC2 Must City Data, Platform + city data + City Data, Platform, + FREQ.9 UC2 Provide mechanisms for static data publication Must + Business Needs + City Data, Platform, + FREQ.10 UC2 Provide mechanisms for real-time data publication Must + Business Needs + + FREQ.11 UC2 Enable the publication of metadata Must City Data, Platform + + + FREQ.12 UC2 Maintain temporal information about the data Must City Data, Platform + + + FREQ.13 UC2 Support sensory data collection Must City Data, Platform + + + FREQ.14 UC2 Accepts content in numerous file types/formats Must City Data, Platform + + Prompts a request for resubmission to the data provider if an error of + FREQ.15 UC2 Must City Data, Platform + data transmission or receipt occurs + + FREQ.16 UC2 Enable the semantic description of connected devices Must City Data, Platform + + + FREQ.17 UC2 Gather data from authenticated and authorized devices Must City Data, Platform + + + FREQ.18 UC2 Validates automatically the successful transfer of the data Must City Data, Platform + + + FREQ.19 UC2 Performs virus checking on data Must City Data, Platform + + Verifies the validity of the submission based on submitter, expected + FREQ.20 UC2 Must City Data, Platform + format, data quality, and completeness + Platform should have built-in checks on the incoming metadata. Data + FREQ.21 UC2 not containing the minimally defined set of attributes should be Must City Data, Platform + returned to the publisher for metadata enhancement. + System should have a user-friendly method of mapping non-standard + FREQ.22 UC2 Should City Data, Platform + metadata elements into approved standard elements. + Once ingested, metadata should be stored in a single common format. + FREQ.23 UC2 This format should be one that ensures against data loss, and allows a Must City Data, Platform + variety of access/distribution options + Data in the repository shall have sufficient technical metadata to + FREQ.24 UC2 assure functionality (e.g. viewing and display) to ensure accessibility Must City Data, Platform + and reusability. + Allows publishers to display and perform manual/visual quality control Business Needs, City + FREQ.25 UC2 Could + assurance via a user-friendly GUI Data, Platform + Business Needs, City + FREQ.26 UC2 Any errors shall prompt a request for resubmission of data Should + Data, Platform + + FREQ.27 UC3 Enable data providers to manage their resources Must Business Needs + + A minimal set of identifying information/metadata concerning data Business Needs, + FREQ.28 UC3 Must + publication submission must be recorded Platform + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 20 + + + Stores and tracks versions of data. Links /connections between + FREQ.29 UC3 Must City Data, Platform + versions are created and maintained + Give service and data providers access to anonymized data of the Business Needs + FREQ.30 UC3 Should + subscribers of their data or services + City Data, Platform, + FREQ.31 UC3 Enable data providers to maintain and repair data and metadata Should + Business Needs + Tracks data publication agreements between Data and Platform Business Needs, + FREQ.32 UC3 Must + Providers Platform + Store terms of agreements, and use them to monitor/review/process + FREQ.33 UC3 Must City Data, Platform + data submissions. + Able to add and edit terms of agreement, based on access of level of Business Needs, + FREQ.34 UC3 Must + user. Platform + + FREQ.35 UC3 Submission volumes and schedules are managed and monitored Must City Data, Platform + + + + +2.4 SUB-GOAL 2: City data is managed in a safe and intelligent manner +Rationale: The urban platform enables users to publish, consume and commercialise data, as well +as deploy and manage services all in a secure and privacy protected manner. + + Achieve [City data is exploited + to its full effect] + Goal + co-ena bles + + + + + Maintain [Resources are managed in a + safe and intelligent manner] + S ub-Goal 1 + + + + + Achieve [Store Data] Achieve [Transmit Data] Achieve [Manage Infrastructure] + UC4 UC5 UC6 + + + Figure 6. Sub-Goal 2 “City data is managed in a safe and intelligent manner” refinement. + + + +Drivers: Ensure data is secured and the identity of users are preserved +Actions: For Sub-Goal 2 to be maintained in the long-run it requires the efficient realisation of use +cases: “Store City Data” and “Retrieve and Transmit City Data”, as shown in Figure 6. + + +2.4.1 Use Case: Store City Data +ID: UC4 +Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner +Pre-condition: Data is successfully published in the platform +Actors: Urban Platform +Rationale: When data is successfully submitted (either via APIs or manual upload), it is +processed/prepared for storage into the platform’s database. This procedure will include the + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 21 + + +generation of unique identifiers to the database, enrichment with ontologies (when applicable), +encrypted (when applicable), signed with digital certificates (when applicable) to ensure that the +data conforms to the platform data formatting, standards, security and regulation. Data may be +converted to accepted formats, as needed (e.g. graph model). A primary goal of the conversion of +content for the platform is the preservation of the content. Priority will be given to preserving the +data accordingly to the policies defined in section (2.6.2). Access-control levels and license models +are associated to data which is subject to restrictions relating to access and conditions of use. +Refines into requirements: FREQ 28 – FREQ 39. + + Use Case Basic Interactions and Responses + 1. Platform mechanisms converts submitted data into a standard format + 2. Security enforcement (e.g. anonymisation, cryptography) is placed on + sensitive information. + UC4. Store City 3. Platform associates with datasets the access-control definitions set by + Data owner of resources. + 4. Data is enriched with semantics and is associated with other datasets + 5. Platform stores data in a scalable and secure database. + 6. End of data storage. + + +2.4.2 Use Case: Transmit Data +ID: UC5 +Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner +Pre-condition: Data is successfully published in the platform +Actors: Urban Platform +Rationale: The platform accepts data retrieval request, validates user’s rights to access the data, +retrieves city data from data storage, and moves a copy of the data to the relevant platform +component for further processing. If special processing is required, the retrieval function accesses +data in staging storage and applies the requested processes. The types of operations, which may +be carried out, include sub-sampling in temporal or spatial dimensions, conversions between +different data types or output formats, and other specialized processing (e.g., data visualisation). +Once it is finalised data will be sent to the appropriate delivery channels (e.g. API’s, GUI). It also +encompasses function to verify corruption during any internal data transfer. This function requires +that all hardware and software within the platform provide notification of potential errors and that +these errors are routed to standard error logs that are checked by the Platform Provider. +Refines into requirements: FREQ 40 to 54. +2.4.3 Use Case: Manage Infrastructure +ID: UC6 +Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner +Pre-condition: The platform is available +Actors: Urban Platform +Rationale: Manage infrastructure provides the services and functions for the overall operation of +the urban platform. Administration functions include monitoring quality of service agreements, +auditing data publication to ensure that they meet archive standards, and maintaining configuration +management of system hardware and software. In overall, it provides system engineering +functions to monitor and improve platform operations, and to inventory, report on, and +migrate/update the contents of the platforms’ databases. +Refines into requirements: FREQ 55 to 62. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 22 + + + + Use Case Basic Interactions and Responses + 1. Platform keeps monitoring services at run-time to ensure operation + and integrity of city data + o If system failure, the platform activates mechanisms for recovery + UC6. Manage based on pre-defined rules + Infrastructure o Platform logs issue and issue alert messages to platform providers + 2. Platform logs operation capabilities (e.g. performance, mean of time + failure, issues, etc.) + + +2.4.4 Functional Requirements + + Req. ID UC. ID Description Priority Domain + A minimal set of identifying information/metadata concerning data Business Needs, + FREQ.28 UC4 Must + publication submission must be recorded. Platform + Stores and tracks versions of data. Links /connections between + FREQ.29 UC4 Must City Data, Platform + versions are created and maintained. + + FREQ.30 UC4 Converts data to accepted file formats Must City Data, Platform + + Keep sensitive information secured and accessible only to authorized + FREQ.31 UC4 Must City Data, Platform + users + + FREQ.32 UC4 Keep user’s personal information protected Should City Data, Platform + + + FREQ.33 UC4 Keep city data and meta-data secured Must Platform Needs + + + FREQ.34 UC4 Enable privacy preserving mechanisms associated to data Must Platform Needs + + + FREQ.35 UC4 Model data in accordance with defined standards Must City Data, Platform + + + FREQ.36 UC4 Support the use of ontologies and semantic modelling of city data Could City Data + + + FREQ.37 UC4 Support database-level provenance annotation Should City Data + + + FREQ.38 UC4 Support data-level provenance annotation Should City Data + + + FREQ.39 UC4 Enable data to be encrypted Should Platform Needs + + System must have the ability to search and display metadata, preferably + City Data, Platform + FREQ.40 UC5 in a user-conformable, human readable display as well as in its native Must + Needs + format for machine harvesting and manipulation. + Controls access to data in the repository based on multiple permission + City Data, Platform + FREQ.41 UC5 levels. These permission levels determine the create/edit/read/delete Should + Needs + privileges granted users. + Access rights and conditions of use will be held for each data and its City Data, Platform + FREQ.42 UC5 Should + related metadata. Needs + Access rights and conditions can be inherited from a parent data to any City Data, Platform + FREQ.43 UC5 Could + data designated as a child data (derived information). Needs + Access rights and conditions of use will be machine readable and City Data, Platform + FREQ.44 UC5 Should + actionable. Needs + Access mechanisms must be sufficiently granular to allow the Business Needs, City + FREQ.45 UC5 identification of individual users, in order to maintain audit logs of actions Should + Data, Platform + performed by users. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 23 + + + Maintains the integrity of the database which contains both metadata + FREQ.46 UC5 and system information. Must Platform Needs + + Provides internal validation such as referential integrity of the contents of City Data, Platform + FREQ.47 UC5 the database. Must + Needs + Creates and maintains schema definitions required to support data + FREQ.48 UC5 management functions. Must Platform Needs + + Monitors and ensures that data and metadata are not corrupted during + FREQ.49 UC5 transfers. Must Platform Needs + + Provides statistically acceptable assurance that no components of the + FREQ.50 UC5 data are corrupted during any internal data transfer. Must Platform Needs + + Performs routine and special data integrity checking for each dataset + FREQ.51 UC5 and generates error reports. Must Platform Needs + + Provides disaster recovery capabilities including data backup, off-site + FREQ.52 UC5 data storage, data recovery, etc. Must Platform Needs + + Refresh/replace data without service interruption, and update + FREQ.53 UC5 corresponding metadata as appropriate. Must Platform Needs + + Ensure that any associated unique identifiers of the updated data are not + FREQ.54 UC5 altered. Must Platform Needs + + Audits submissions to ensure that they meet archive/repository + FREQ.55 UC6 standards. Must Platform Needs + + Maintains configuration management of the system hardware and + FREQ.56 UC6 software. Must Platform Needs + + Has capability to inventory, report on and migrate the contents of the + FREQ.57 UC6 repository. Must Platform Needs + + + FREQ.58 UC6 Ensures data integrity for version upgrades and format migration. Must Platform Needs + + + FREQ.59 UC6 Monitors functionality of the entire repository. Must Platform Needs + + + FREQ.60 UC6 Maintains integrity of system configuration. Must Platform Needs + + Audits system operations, performance and usage. + FREQ.61 UC6 Must Platform Needs + + Provides platform performance information and database holdings + FREQ.62 UC6 inventory reports Must Platform Needs + + + + +2.5 SUB-GOAL 3: City data is orchestrated in a marketplace +Rationale: The urban platform enables users to consume and publish data in a secure and privacy +protected manner. +Drivers: Ensure data is secured and the identity of users are preserved +Actions: For Sub-Goal 3 to be maintained in the long-run it requires the efficient realisation of use +cases: “Commercialise City Data” and “Commercialise Data Services”, as shown in Figure 7. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 24 + + + + Achieve [City data is exploited + to its full effect] + Goal + + + + + co-ena bles + Maintain [City data is orchestrated in + a marketplace] + S ub-Goal 3 + + + + + Achieve [Commercialise Data + Achieve [Commercialise City Data] + Services] + UC7 UC8 + + + + + Figure 7. Sub-Goal 3 “City data is orchestrated in a marketplace” refinement. + + + +2.5.1 Use Case: Commercialise City Data +ID: UC7 +Refines: SUB-GOAL 3: City data is orchestrated in a marketplace +Pre-condition: Data is successfully published in the platform, both publisher and consumers of +city data are authenticated in the platform, and there are billing capabilities available. +Actors: Data Publishers, Data Consumers +Rationale: The providers of city data can commercialise city data based on the policies and +financial models defined in the platform. After publishing their data, publishers can define which +data can be available as open data and which data should be available with the payment of a +subscription fee. Once publishers define which data is to be commercially exploited, the platform +will associate the data with their respective financial models and let it ready for subscription. City +data consumer chooses which data to purchase and is redirected to a billing interface where the +subscription payment is taken. The platform must provide an update response indicating the status +of the payment. If successful, data is ready available to be consumed by humans and machines, +otherwise the user can re-try the payment or cancel transaction. +Commercialise city data also involves the function of managing commercial data. It provides +services and functions for updating, maintaining and accessing both data and its respective +commercial transactions. Furthermore, it enables data providers to track the usage of commercial +data by users. Ideally the owners of the data should be the only authorised user to manage +resources, and other authorised users can track the usage of the data in the platform. Data usage +tracking includes performing queries on the data management data to generate result sets, and +producing reports from these result sets. Data consumers are provided with functions which enable +them to manage their subscriptions and financial transactions on the platform. These functions +include updating, maintaining and accessing financial transactions. For all these functions and +services, the platform must provide a database update response indicating the status of the +update, avoid update errors to be propagated in the platform, and should keep an audit trail of all +actions to enable rollback. +Specialised Use Cases: The Use Case Commercialise City Data is distinguished into four +specialised Use Cases: “Commercialise Data (UC7.1)”, “Consume Commercial City Data (UC7.2)”, +“Manage Commercial Data (UC7.3)” and “Manage Data Subscription (UC7.4)”. +Refines into requirements: FREQ 63 to 68. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 25 + + + + + Use Case Basic Interactions and Responses + 1. User selects the data to be commercialised + 2. User selects the commercial model for data consumption + UC7.1. 3. Platform validates selection + Commercialise 4. Platform associates data to subscription model + Data 5. Platform releases data for commercial exploitation in the marketplace + 6. End of data commercialisation set up. + 1. User selects the data to be subscribed to + 2. User request subscription to data + UC7.2. 3. Platform validates selection + Consume 4. Platform redirects user to billing system + Proprietary 5. Billing system deals with user request + Data o If successful, user is redirected to a GUI where data is ready to use + o If unsuccessful, user can try payment again or cancel request + 2. End of data subscription. + 1. Platform provides user with an interface for commercial data + management + 2. User chooses to edit or delete commercial data + o If edit, user revise commercial models, licenses, access-control, + semantics; + o If delete, user selects dataset(s) to be removed (following policies + UC7.3. Manage defined in the platform for data removal) + commercial 3. User confirms action + Data 4. Platform promptly process user’s request + 5. Platform confirms execution of request + o If valid request, platform acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 6. End of resources management + 1. Platform provides user with an interface for data subscription + management + 2. User chooses to edit or cancel data subscription + o If edit, user revise payment and subscription timeframe; + o If cancel, user selects dataset(s) to have subscription cancelled + UC7.4. Manage (following policies defined in the platform for data subscription) + data 3. User confirms action + subscription 4. Platform promptly process user’s request + 5. Platform confirms execution of request + o If valid request, platform updates acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 6. End of data subscription management + + + +2.5.2 Use Case: Commercialise Data Services +ID: UC8 +Refines: SUB-GOAL 3: City data is orchestrated in a marketplace +Pre-condition: Data is successfully published in the platform, both publisher and consumers of +city data are authenticated in the platform, and there are billing capabilities available. +Actors: Data Service Providers, Data Publishers and Consumers + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 26 + + +Rationale: After deploying data services in the platform, the providers of data services can choose +to commercialise services based on the policies and financial models defined in the platform. Once +data service providers define which service(s) is (are) to be commercially exploited, the platform +will associate the services with their respective financial models and let available in the platform +applications module ready for use. User (either publisher or consumer) chooses which data +services to use, and in case a charged service is selected the platform redirects the user to a +billing interface where the payment is taken. The platform must provide an update response +indicating the status of the payment. If successful, service is ready available to be used, otherwise +the user can re-try the payment or cancel transaction. Note that data service owners should be +able to waive the payment of tariff to certain users’ categories. +Data services providers are offered with functions to manage their commercial services. The +platform provides functions for updating, maintaining and accessing both service and its respective +commercial transactions. Furthermore, it enables data services providers to track the usage of +services by users. Services usage tracking includes performing queries on the platform to +generate result sets, and producing reports from these result sets. The consumers of data services +are provided with functions which enable them to manage their subscriptions and financial +transactions. These functions include updating, maintaining and accessing financial transactions. +For all these functions and services, the platform must provide a database update response +indicating the status of the update, avoid update errors to be propagated in the platform, and +should keep an audit trail of all actions to enable rollback. +Specialised Use Cases: The Use Case Commercialise Data Services is distinguished into four +specialised Use Cases: “Commercialise Data Services (UC8.1)”, “Consume Data Services +(UC8.2)”, “Manage Commercial Services (UC8.3)” and “Manage Services Subscription (UC8.4)”. +Refines into requirements: FREQ 68 to 73. + + Use Case Basic Interactions and Responses + 1. User selects the data services to be commercialised + 2. User selects the commercial model for data service usage based on the + category of users + UC8.1. 3. Platform validates selection + Commercialise 4. Platform associates data services to subscription model + Data Services 5. Platform enables data service to be commercially exploited in the + marketplace + 6. End of services commercialisation set up. + 1. User selects the data service to be subscribed to + 2. User request subscription to service + 3. Platform validates selection + UC8.2. 4. Platform validates user’s category its respective commercial models + Consume Data 4. If applicable, platform redirects user to billing system + Services 5. Billing system deals with user’s request + o If successful, user is redirected to a GUI where data is ready to use + o If unsuccessful, user can try payment again or cancel request + 6. End of data subscription. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 27 + + + 1. Platform provides user with an interface for commercial data + management + 2. User chooses to edit or delete commercial data + o If edit, user revise commercial models, licenses, access-control, + semantics; + o If delete, user selects dataset(s) to be removed (following policies + UC8.3. Manage defined in the platform for data removal) + commercial 3. User confirms action + services 4. Platform promptly process user’s request + 5. Platform confirms execution of request + o If valid request, platform acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 6. End of resources management + 1. Platform provides user with an interface for data subscription + management + 2. User chooses to edit or cancel data subscription + o If edit, user revise payment and subscription timeframe; + o If cancel, user selects dataset(s) to have subscription cancelled + UC8.4. Manage (following policies defined in the platform for data subscription) + services 3. User confirms action + subscription 4. Platform promptly process user’s request + 5. Platform confirms execution of request + o If valid request, platform updates acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 6. End of data subscription management + + +2.5.3 Functional Requirements + + Req. ID UC. ID Description Priority Domain + City Data, Platform, + FREQ.63 UC7 Support the commercialization of city data Should Business Needs + City Data, Platform, + FREQ.64 UC7 Enable users to subscribe to city data through the payment of a tariff Should Business Needs + City Data, Platform, + FREQ.65 UC7 Enable users to manage their data subscriptions Should Business + Provide platform providers mechanisms to define the terms and + FREQ.66 UC7 Must Platform Needs + conditions for platform data usage + Enable data providers to manage the subscription models of their City Data, Platform, + FREQ.67 UC7 Should + data Business + UC7/ City Data, Platform, + FREQ.68 Utilise secure and reliable billing and payment management systems Must + UC8 Business + City Data, Platform, + FREQ.69 UC8 Support the commercialization of data services Should Business Needs + Enable data providers to manage the commercial models of their City Data, Platform, + FREQ.70 UC8 Should + services Business Needs + Provide service providers mechanisms to define the terms and City Data, Platform, + FREQ.71 UC8 Should + conditions of platform services Business Needs + Allow users to pay a tariff for using certain advanced services (e.g. City Data, Platform, + FREQ.72 UC8 Should + Data manipulation, enrichment) Business Needs + City Data, Platform, + FREQ.73 UC8 Enable users to manage their data services subscriptions Should Business Needs + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 28 + + +2.6 SUB-GOAL 4: City data is offered in an accessible manner +Rationale: The urban platform provides city data in both human and machine (e.g. sensors, +actuators, systems) readable and understandable formats. +Drivers: Ensure data understandability and machine-to-machine data transaction. +Actions: For Sub-Goal 3 to be maintained in the long-run it requires the efficient realisation of use +cases: “Register Consumer”, “Discover City Data”, and “Consume City Data” as shown in Figure 8. + + Achieve [City data is exploited + to its full effect] + Goal + + + + + co-ena bles + Maintain [City data is offered in an + accessible manner] + S ub-Goal 1 + + + + + Achieve [Register Consumer] Achieve [Discover City Data] Achieve [Consume City Data] + UC9 UC10 UC11 + + + + + Figure 8. Sub-Goal 4 “City data is offered in an accessible manner” refinement. + + + +2.6.1 Use Case: Register Data Consumer +ID: UC9 +Refines: SUB-GOAL 4: City data is offered in an accessible manner +Pre-condition: User is not logged in the platform +Actors: Data Consumers +Rationale: Data Consumers can register in the platform and request approval to consume city data +via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to +confirm their registration. Users must accept the terms and conditions of platform usage and define +how their personal data can be used by the Platform Owner. Users can manage and alter their +registration information at any time they want to. +Refines into requirements: FREQ 64. + + Use Case Basic Stimulus and Responses + 1. The platform prompts the user for a username and password or register + new account. + 2. The user selects registration option. + 3. The platform prompts user for data consumer registration information + UC1. Register 4. The user enters in their information. + Consumer 5. Platform verifies information and creates account. + o If non-valid information, platform shows error message and returns to + step 1. + 6. Platform acknowledges registration has been successful + 7. End of registration + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 29 + + +2.6.2 Use Case: Discover City Data +ID: UC10 +Refines: SUB-GOAL 4: City data is offered in an accessible manner +Pre-condition: User has access to either platform GUI or API +Actors: Data Consumers +Rationale: Data Consumers can register in the platform and request approval to consume city data +via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to +confirm their registration. +Refines into requirements: FREQ 64. + + Use Case Basic Stimulus and Responses + 1. Users access specialised data query end-points (e.g. SPARQL) + 2. Users provides information for pre-defined parameters for search + 3. Users request data search + 4. Platform quickly process users request for data + UC10.1 o All queries are verified against access rights restrictions + Discover city o If restriction applies users are redirected to log in interfaces + data via data o Users provide credentials and log on the system + query end-points 5. Users are provided with query results on the end-point if access is + allowed + o If access is not not allowed platform issues an error message to the + user + 1. Users search city data via GUI + 2. Users inputs search parameters (e.g. key words, categories, formats, + publishers) + UC10.2 3. Users request data search + Discover city 4. Platform quickly process users request for data + data via data o All queries are verified against access rights restrictions + query end-points o If restriction applies users are redirected to log in interfaces + o Users provide credentials and log on the system + 5. Users are provided with query results on an interface + o If access is not allowed platform issues an error message to the + user + + + +2.6.3 Use Case: Consume City Data +ID: UC11 +Refines: SUB-GOAL 4: City data is offered in an accessible manner +Pre-condition: User has access to either platform GUI or API +Actors: Data Consumers +Rationale: Data Consumers can register in the platform and request approval to consume city data +via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to +confirm their registration. +Refines into requirements: FREQ 64. +Subordinated Use Cases: “Consume proprietary city data” + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 30 + + + + Use Case Basic Stimulus and Responses + 1. Users request data to be formatted in a particular format supported by + UC11.1 the platform + Customise City 2. Platform quickly process users request for data formatting + Data 3. Mechanism for data conversion is called and process data + 4. Users are provided with data formatted as requested + 1. Users / Machines select data to be downloaded + 2. Users / Machines are redirected to authentication mechanism in case of + UC11.2 registration is needed for the particular dataset + Consume City o If authentication is successful, users are provided with requested + Data via GUI data streams + 3. Users are provided with requested data via APIs + 1. Users / Machines makes data request on the platforms API + 2. Users / Machines are redirected to authentication mechanism in case of + UC11.3 registration is needed for the particular dataset + Consume City o If authentication is successful, users are provided with requested + Data via APIs data streams + 3. Users are provided with requested data via APIs + + + +2.6.4 Functional Requirements + + Req. ID UC. ID Description Priority Domain + Allow users to register to use services and consume proprietary city Societal Needs, + FREQ.74 UC9 Should + data and open data (optional) Platform + Keep sensitive information secured and accessible only to Societal Needs, + FREQ.75 UC9 Should + authorized users Platform + Societal Needs, + FREQ.76 UC9 Provide authentication mechanisms for users Must Platform + Societal Needs, + FREQ.77 UC9 Keep user’s personal information protected Must Platform + Allow users to control which data they are willing to provide and how Societal Needs, + FREQ.78 UC9 Must + their data should be used Platform + Societal Needs, + FREQ.79 UC10 Allow users to format data in any supported data formats Must + Platform + The query request may require data to be sourced from different Societal Needs, + FREQ.80 UC10 Must + storage locations Platform + Allows query requests against all metadata used to manage the Societal Needs, + FREQ.81 UC10 repository. Should Platform + Provide users information about the legal aspects of the data Societal Needs, + FREQ.82 UC11 Must + (license, ownership) City Data + Societal Needs, + FREQ.83 UC11 Keeps an audit trail of all actions. Must City Data + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 31 + + +2.7 SUB-GOAL 5: User’s experience is enhanced by the provision of +value-added services +Rationale: The urban platform enables users to consume and publish data in a secure and privacy +protected manner +Drivers: Ensure data is secured and the identity of users are preserved +Actions: For Sub-Goal 5 to be maintained in the long-run it requires the efficient realisation of use +cases: “Deploy Data Services”, “Manage Services”, and “Utilise Data Services” as shown in Figure +8. + + Achieve [City data is exploited + to its full effect] + Goal + + + + + co-ena bles + Maintain [User s experience is enhanced by + the provision of value-added services] + S ub-Goal 1 + + + + + Achieve [Deploy Data Services] Achieve [Manage Services] Achieve [Utilise Data Services] + UC12 UC13 UC14 + + + + + Figure 9. Sub-Goal 5 “User’s experience is enhanced by the provision of value-added services” refinement + + + +2.7.1 Deploy Data Services +ID: UC12 +Refines: SUB-GOAL 5: User’s experience is enhanced by the provision of value-added services +Pre-condition: Data Services Providers are provided with credentials, are authorised to deploy +their services in the platform, and have access to technical specifications of the platform interfaces. +Actors: Data Services Providers +Rationale: Data Services can register in the platform and request approval to deploy services in +the platform. They provide valid registration details (to be defined) and wait for registration +confirmation. Platform Providers may authorise or not data services providers to offer both open +and proprietary services in the platform. Data services providers must formally agree with service +deployment agreement with the Urban Platform. This agreement defines terms of the content, +policies, regulations, license agreement. The Urban Platform will proactively work with Service +Providers to agree on the technical specifications of interfaces and platform openness level. +Agreements between Platform and Data Service Providers may be renegotiated on a periodic or +ad-hoc basis. +Refines into requirements: FREQ.1 to FREQ.5. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 32 + + + + Use Case Basic Stimulus and Responses + 1. User authenticates in the platform + 2. User is provided access to platform interfaces for service deployment + 3. User deploy services + 4. Platform checks compatibility and any technical issues arising from the + UC12. Deploy new service + Services 5. If approved the service is ready to be used and managed in the + platform + o If deployment is not approved, platform shows error message and + returns to step 1. + 6. End of deployment + + + +2.7.2 Use Case: Manage Services +ID: UC13 +Refines: GOAL 5: User’s experience is enhanced by the provision of value-added services +Pre-condition: User successfully authenticates in the platform +Actors: Data Services Providers +Rationale: Manage services provides the functions for updating, maintaining and accessing +services as well as tracking their usage by users. Ideally the owners of the services should be the +only authorised user to manage them, and other authorised users can track the usage of the +services in the platform. In case of updates the platform must log in the database update details +and send to service providers a response indicating the status of the update. The platform should +also ensure update errors are not propagated nor affect the health of other services provided in the +platform, and should keep an audit trail of all actions to enable rollback. Data services usage +tracking includes performing queries on the data management data to generate result sets, and +producing reports from these result sets. All user’s information provided to service providers must +follow regulations of data protection and the user’s defined rules for their data use. +Specialised Use Cases: The Use Case Manage Services is distinguished into two specialised +Use Cases: “User manages services (UC13.1)” and “User tracks service usage (UC13.2)”. +Subordinated Use Cases: “Transmit Data (UC5)” +Refines into requirements: FREQ.27 to FREQ.25. + + + Specialised Use Cases Basic Interactions and Responses + 1. Platform provides user with an interface for services management + 2. User chooses to edit or delete services + 3. If edit, user revise service information (access-control, + commercial models, parameters) and deployment; + If delete, user selects services to be removed / disabled + UC13.1. User manages 4. User confirms action + services 5. Platform quickly process user’s request + 6. Platform confirms execution of request + o If valid request, platform acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 7. End of services management + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 33 + + + 1. Platform provides user with an interface for services management + 2. User chooses to visualise usage information of a service + 3. Platform quickly process user’s request for data usage + UC13.2. User tracks information + services usage 4. Platform provides user with statistical information about services + usage and data users anonymised information + 5. End of data services tracking. + + + +2.7.3 Use Case: Utilise Data Services +ID: UC2 +Refines: SUB-GOAL 1 - City data is collected in an intelligent manner +Pre-condition: User is authenticated in the platform +Actors: City data publisher +Rationale: Data Consumers can register in the platform and request approval to consume city data via GUI +or APIs. They provide valid registration details (to be defined) and wait for platform to confirm their +registration. +Subordinated Use Cases: “Commercialise Data Services (UC8)” +Refines into requirements: FREQ 6 to FREQ.26. + + + Specialised Use Cases Basic Interactions and Responses + 1. Users / Machines select data service to be utilised + 2. Users / Machines are redirected to authentication mechanism in + UC2.1. User utilises data case of registration is needed for the particular service + services o If authentication is successful, users are provided with + requested data streams + 3. Users are provided with requested service either via API or GUI + + +2.7.4 Functional Requirements + + Req. ID UC. ID Description Priority Domain + Provide stable and well-defined interfaces to ensure interoperability + Societal Needs, + FREQ.84 UC12 between the platform, services and the applications provided by Should Platform + services providers + Ensure the interfaces of the architecture are open to reduce entry Societal Needs, + FREQ.85 UC12 Should + barriers and integration issues Platform + Provide multi-purposed and network intelligent interfaces to Societal Needs, + FREQ.86 UC12 Must + providers and consumers of services Platform + Provide service providers mechanisms to define the terms and Societal Needs, + FREQ.87 UC12 Must + conditions of platform services deployment Platform + Business Needs, + FREQ.88 UC13 Provide statistical information of user’s feedback on service usage Must Platform + Allow users to use services to manipulate city data (e.g. Create Societal Needs, + FREQ.89 UC10 Must + mash ups, integrate) Platform + Allow users to provide feedback on usability, and quality of data Societal Needs, + FREQ.90 UC14 Must + and services provided by the platform Platform + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 34 + + + +3. Non-functional Requirements +3.1 Run-time Quality Requirements +3.1.1 Scalability Requirements + +Rationale: The ability of the system to execute its task within its expected performance profile and +to handle on-demand increased processing volumes of data and service requests +Drivers: Provide ready access to all data that underpins decision making processes in smart cities, +and accommodate users and data usage patterns +Refines into requirements: NFREQ.1, NFREQ.2, NFREQ.3, NFREQ.4 +Relevance: Urban platforms have ambitious performance requirements. Such platforms must cope +with user’s demand for data and services, capture real-time data that will be catalysed by a myriad +of sensors. The demand for urban platform is very likely to significantly increase over time. It is +very difficult to have clear performance characteristics due to the ubiquity, heterogeneity high +connectivity of devices and end users. + + SCALABILITY MEASURES + Actions Capture the performance requirements + Create service level agreements + Predict scalability using software simulation + Analyse the performance of the platform overtime + Conduct practical testing + Strategy Prioritize service and data requests + Distribute processing over time + Scale up or scale out as necessary + Reuse resources and results Partition and parallelize + Constantly monitor Quality of Service at runtime + + + +3.1.2 Availability and Reliability Requirements + +Rationale: The ability of the system to be fully or partly operational as and when required and to +effectively handle failures that could affect system availability +Drivers: Build a reliable foundation for “on demand” exploitation of data +Refines into requirements: NFREQ.5, NFREQ.6, NFREQ.7, NFREQ.8 +Relevance: Any system that has complex or extended availability requirements, complex recovery +processes, or a high profile (e.g., is visible to the public) + + AVAILABILITY AND RELIABILITY MEASURES + Actions Capture the availability requirements Produce the availability schedule + Estimate platform availability Estimate functional availability Assess against the + requirements Rework the architecture + Strategy Adopt fault-tolerant hardware + Use reliable infrastructure database + Log transactions + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 35 + + + Develop adaptive software to cope and recover from faults + Design and test for failure + Deploy load balancing + Identify suitable backup and disaster recovery solution + + +3.1.3 Trust Requirements + +Rationale: A quality related to the user’s belief in the reliability, integrity and ability of the functional +behaviour of the platform +Drivers: Gain understanding of what influences user’s experience while interacting with services +provided +Requirements: NFREQ.16, NFREQ.17 +Relevance: Relevant to the systems that share and collect information that may raise public +concern. In some cases, trust has to do with the reliability of data and their providers, whereas in +other cases trust can be associated with the security and privacy of the technology that was +deployed. Trust affects the reputation of the platform besides its dissemination and maturity on the +market. + +TRUST MEASURES + Capture trust requirements + Perform risk analysis so measures can be implemented +Actions Explore the vulnerability aspects of city data + Check whether extensibility requirements impact on trust + Define a trust model + Adopt trust model + Deploy monitoring capabilities and assess its impact on scalability + Manage the data in a way that ensures its compliance with data +Strategy protection regulations + Implement tampering and data misuse detection + Make use of Cryptography when necessary + Allow Federation of trust between platforms + + + +3.1.4 Security Requirements + +Rationale: Ability of the system to enforce the intended confidentiality, trust, integrity and service +and data access policies, and to detect and recover from failure in these security mechanisms. +Drivers: Manage the data and services in a way that ensures its integrity, and compliance with +data protection regulations +Relevance: Relevant to the systems that share and collect information that may raise public +concern. Urban platforms may become a valuable target for attackers which can potentially leave +huge swathes of information exposed. It could potentially undermine trust in the government and +damage its reputation. +Refines into requirements: NFREQ.9 to NFREQ.15 + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 36 + + + SECURITY MEASURES + Elicit the security requirements + Cross-check security requirements’ impact on services offered by different + providers + Verify security impact on service composition + Conduct risk analysis and impacts of security breach + Use scalable and efficient user authentication components + Use authentication and authorization components to secure interfacing with + Actions external services + Address aspects of data collection, storage and distribution + Address aspects of service and communication security among devices + Identify security requirements of the physical infrastructure + Balance and prioritize scalability (performance) and security requirements + Evaluate trade-offs between privacy considerations and preventing abuse. (While + anonymous access guarantees privacy of users, traceability of users due to abuse + of the service is not possible.) + Toughen user’s functional components + Authenticate subjects + Define and enforce access policies + Secure communication infrastructures + Secure interfaces with external systems and services + Strategy + Secure databases + Check data integrity for critical services + User digital certificates and encryption where necessary + Secure monetary transactions + Secure user’s personal information + + +3.1.5 Privacy Requirements + +Rationale: Ability of the system to ensure that the collection and transmission of personal data is +minimized and handled in accordance with user’s expectation and regulations. +Drivers: Protect the vulnerability aspects volunteered citizen’s data +Requirements: NFREQ.18, NFREQ.19, NFREQ.20, NFREQ.21, NFREQ.22 +Relevance: Ensuring users privacy is protected positively influences user’s experience, +acceptance and continuous use of the platform. Besides other factors, the reputation of the +platform depends on how well user’s information is secure and preserved. + + PRIVACY MEASURES + Capture trust requirements + Perform risk analysis so measures can be implemented + Actions Explore the vulnerability aspects of city data + Check whether extensibility requirements impact on trust + Define a trust model + Allow users to interact with the platform anonymously in given + circumstances + Manage users identification securely + Strategy Anonymize data whenever necessary to comply with data regulations + Cryptograph personal identifiers during data transmission when that is + necessary + Avoid unauthorized access to implicit information (e.g. location) + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 37 + + + Verify the impact of security, trust and scalability requirements trade-offs + on privacy + Allow the user to control how personal information is used + Empower user to control the data disclosure mechanism + + + +3.2 Non Run-time Quality Requirements +3.2.1 Evolvability Requirements + +Rationale: The ability of the platform to withstand and easily adapt when new requirements and +changes is introduced. +Drivers: Ensure the platform is able to accommodate additional functionality and emerging +technologies at later stage at a fair and transparent cost +Refines into requirements: NFREQ.23 +Relevance: Important for longer- lived and more widely used systems. Urban platforms are +expected to be highly evolvable in order to accommodate future emerging technologies and avoid +interoperability issues. + + EVOLVABILITY MEASURES + Actions Characterize and assess the evolution needs + Consider the evolution trade-offs + Balance and negotiate potential conflicting requirements emerging from + the evolution capability. + Strategy Adopt standards and open interfaces + Design loose-coupled components + Preserve the platform resilience + + + +3.2.2 Extensibility Requirements + +Rationale: The flexibility of the system to allow services and functionality to be extended and +augmented by service providers in order to increase value of services to both platform providers +and end-users. The extension of the platform services is determined by the Platform Openness +strategy. +Drivers: Stakeholders can extend the services provided by the urban platform, so that +partnerships can be built to deliver holistic and interoperable solutions. Identify integrated +approaches to design and service delivery which ensures that services fit together and that +synergies can be exploited. +Refines into requirements: NFREQ.24, NFREQ.25 +Relevance: Important for longer- lived and more widely used systems. Urban platforms are +expected to be highly evolvable in order to accommodate future emerging technologies and avoid +interoperability issues. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 38 + + + EXTENSIBILITY + Actions Characterize and assess the extensibility needs + Trace the impacts and cost of extensions + Negotiate potential conflicting requirements emerging from adding extensions + in the platform + Strategy Develop a modular architecture with standard interfaces that reduces entry + barriers due to increased transparency and integration + Share technical information about interfaces will help service providers in + targeting opportunities around the platform + + + +3.3 List of Non-Functional Requirements + +ID # Description Concern Priority Domain + + Platform, +NFREQ.1 Support different service level agreements (SLA) Scalability Should + Infrastructure + Platform, +NFREQ.2 Process services and events on a set of distributed nodes Scalability Should + Infrastructure + Platform, +NFREQ.3 Continuously monitor Quality of Service at runtime Scalability Should + Infrastructure + Platform, +NFREQ.4 Balance its load at runtime Scalability Should + Infrastructure + Platform, +NFREQ.5 Provide high availability Availability Should + Infrastructure + Platform, +NFREQ.6 Guarantee infrastructure availability Availability Should + Infrastructure + Platform, +NFREQ.7 Ensure network availability Availability Should + Infrastructure + Platform, +NFREQ.8 Be able to perform self-healing Availability Should + Infrastructure + City Data, +NFREQ.9 Expose data and services to authorized users Security Must + Platform + City Data, +NFREQ.10 Ensure services are always accessible to entitled users Security Must + Platform + +NFREQ.11 Ensure Data Freshness Security Must Platform + +NFREQ.12 Support access control mechanisms Security Must Platform + + Platform, +NFREQ.13 Have security mechanisms to protect data transmission Security Should + Infrastructure + Platform, +NFREQ.14 Make it difficult to spy on communicated messages Security Should + Infrastructure + Platform, +NFREQ.15 Be able to perform to detect threats at runtime Security Should + Infrastructure + Provide trusted and secure communication and information Platform, +NFREQ.16 Trust Should + management Infrastructure + +NFREQ.17 The platform infrastructure and services shall be trustable Trust Should Infrastructure + + Societal Needs, +NFREQ.18 Allow users to use free services anonymously Privacy Should + Platform + Societal Needs, +NFREQ.19 Allow people to use free services anonymously Privacy Should + Platform + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 39 + + + Allow users to control which data they are willing to provide Societal Needs, +NFREQ.20 Privacy Should + and how their data should be used Platform + +NFREQ.21 Keep users access-control rights/ policies secured. Privacy Should Platform + + Provide privacy protection for users interacting with the Societal Needs, +NFREQ.22 Privacy Should + platform Platform + Platform, +NFREQ.23 Provide communication confidentiality Privacy Should + Infrastructure + Societal Needs, +NFREQ.24 Be extensible for future technologies. Evolvability Must City Data, Platform, + Infrastructure + Platform, +NFREQ.26 Provide standard interfaces for service providers Extensibility Should + Business Needs + +NFREQ.26 Be able to provide services in an interoperable manner Extensibility Should Platform + + + + +4. Other Requirements +4.1 Minimal Descriptive Metadata Required from Data Provider + +Minimal Descriptive Metadata Required from Data Provider + + Attribute Required Definition Notes Examples + +Database Name Y A name given to the resource + +Author Name of a person or body associated with + Y + the creation of the resource + +Maintainer Name of the entity responsible for making + Y + the resource available + +Date Created Y Date the dataset was created + + +Date Modified Y Date the dataset was modified + + +Last Revision Y Date the dataset was last revised + + +Update Frequency Y Frequency of data maintenance + + +Mode of Release Y Open/Proprietary + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 40 + + +4.2 Urban Platform Supported Data Formats + + +MIME type Description Extensions Level + +application/json JavaScript Object Notation json supported + +application/xml eXtensible markup language file xml supported + +text/csv Comma separated values file csv supported + + + + +5. Conclusion & Forward Plans +This document represents the first set of requirements specification for urban platform. Future +activities will collaboratively assess, resolve requirements conflicts, prioritize, and validate the +requirements of the urban platform. Ultimately, this document will become a complete final +requirements specification document to guide and speed up the development open platform for +cities. Table below shows the milestones, deliverables and engagement activities completed and +yet to be completed by the Demand Side Engagement Stream. + + + + Milestones, deliverables and engagement activity Forecast date Status + Stakeholders invitation Late Completed + September/2015 + Online Workshop 1: Project Scope and Outcomes 20/11/2015 Completed + Online meeting held with city members to communicate goals, + expected outcomes, time commitment and required actions. + Start of Collaborative Requirements Engineering Process 20/11/2015 Completed + Participants reviewed the first draft of urban platform requirements, + provided comments, and suggested new requirements and changes. + Online Workshop 2: Review and Refine Requirements 08/12/2015 Completed + Participants had a conference call to review the requirements + specification document and provide their comments. + Deliverable: Requirements Specification Document v2.1 shared 05/01/2016 Completed + across the Working Streams for consultation + Online Workshop 3: Review Requirements Specification 12/01/2016 Completed + Document and discussion of Letter of Intent + Participants had a conference call to review the requirements + specification document and provide their comments, and discussed + the Letter of Intent to be signed by Mayor/Equiv of EU cities to + commit to application of common U.P. approach. + Urban Platforms Workshop in Brussels 19/01/2016 Completed + Participants provided feedback on the requirements specification + document and recommended changes in the document + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 41 + + + Deliverable: Updated Requirements Specification Document v2.2 29/01/2016 Completed + shared with Industry and Standardisation Working Streams + Deliverable: Online release of the Requirements Specification Early + Document and open calls for wider consultation on the Requirements February/2016 + + Online Workshop 4: Engagement with Industry Side Early + Capture industry feedback on the Requirements Specification February/2016 + Document to assess its suitability to drive the development of the first + draft reference architecture. + Online Workshop 5: Review Requirements Specification Mid + Document v2.2 and discuss requirement prioritization and February/2016 + balancing + Collectively determine which candidate requirements of urban + platforms should be prioritized as high importance to all the cities + (use of Value Oriented Prioritization Method). Requirements are also + prioritized to minimize risk during development of urban platforms so + that the most important requirements are made explicit and + considered in the reference architecture. + Capture of case studies from cities intended to support requirements Mid March/2016 + validation (City of Porto), and learning and confidence building + amongst cities. + \ No newline at end of file diff --git a/files/core_requirements_part1.txt b/files/core_requirements_part1.txt new file mode 100644 index 0000000..0e3409e --- /dev/null +++ b/files/core_requirements_part1.txt @@ -0,0 +1,250 @@ + + + Sub-Goal 5: User’s experience is + enhanced by the provision of Use Cases: 3 Requirements: 8 + value-added services + + + + Figure 2 Structure of this document. + + + + +1.6 Product Scope and Perspective +The EIP Project’s urban platform is an open common architecture which serves for city data +collection, management and distribution. An urban platform is intended to support the widespread +exploitation of city data by humans and machines in the urban environment. Figure 3 illustrates a +holistic high level overview of the urban platform the EIP project intends to deliver. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 6 + + + +The reference architecture for urban platforms should: + +• Cater for interoperability between urban infrastructures +• Enable replicability of the solutions/platforms city to city +• Scale without technical constraints and excessive cost increase +• Provide open APIs and SDKs +• Enable Real Time capabilities +• Support implementation of functional and technical capabilities + + + + + Figure 3. High level overview of the urban platform (currently approved EC DG CNECT). + + +The current urban platform market is nascent. Many software vendors offer such a platform, though +many requirements and expectations of the stakeholders of city data are not (fully) addressed. As +a result, current urban platforms are often more costly to design and maintain, less reusable and +often not interoperable platform-to-platform, and susceptible to information fragmentation and +overload. + +The urban platform which the EIP initiative intends to design takes a step beyond the platforms +currently on the market by ensuring the requirements are fully founded on a co-created and +common set of representative city needs, from which it solicits suitable industry input, and an open +and managed collaboration between industry, cities and communities, and others, in order to take +into account their needs and concerns. To do this, it is necessary to take a technology agnostic +approach to design an open and common reference architecture for urban platforms. This platform +must ensure data is collected and sustained in accordance with well-stablished standards, +managed in a robust manner so that it can handle high level supply and demand of data, and +distributed across different value chains, systems and stakeholders. The ability to handle high level +of city data supply and demand while being user secure and accessible enough for city-wide +exploitation of data is one of many keys to the success of urban platforms. This is central to the +design and implementation of urban platforms. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 7 + + +1.7 The Urban Platform Development Stack +The requirements specification of this document is based on the Urban Platform Development +Stack illustrated in Figure 4. The stack is composed by five domains (represented as layers in the +stack) necessary to fully implement an urban platform software suite as shown in Table 1. Each +domain comprehends a set of requirements necessary to the design of a common and open urban +data service platform. The elicited requirements are used to define a technical architecture which is +simple enough to be comprehensible at least at a high level of abstraction. The platform should be +conceptually decomposable into its major subsystems, the platform’s functionality reused by many +services and external applications should be identifiable, and interactions between the platform +and services, data providers and data consumers should be well defined and explicit. + +The first layer of the stack “Societal needs” concerns to outcomes we strive for within a portfolio of +city service domains. An urban platform should recognise societal needs and wants as the starting +point for city data service offering. Ultimately, an urban platform aims to provide tailor made and +compelling engaging services for the users. The Services and Business models layers concerns +with delivering data services which carefully targets the needs and expectations of the different +users of the urban platform, and explore use cases and commercial models where data is used to +deliver different forms of value. The city data layer concerns with the mechanisms necessary to +transform urban platforms into a foundation for widespread exploitation of data, including handling +data architectural features, data usability, semantics and quality aspects. The urban platform layer +concerns to the technology foundation to configure, share, and interpret exponentially increasing +volumes city data and services. Finally, the Infrastructure layer concerns with the base level +connectivity that supports the platform to be scalable and reliable in the long run. + + + + STACK OUTPUT + + Requirements to deliver new digital services that will address the + Societal Needs societal needs of cities in a positive manner that relates to political + narratives. + + + Services & Requirements to new profitable business models and the development + Business Models of an increase range of new and engaging services in the smart cities. + + + + City Data Requirements to provide all city data stakeholders ready access and + delivery of all city data that unpins the decision making process in smart + cities. + + + Urban Platform Requirements to put in place applications together to build a foundation + for the widespread exploitation of data. + + + Infrastructure Requirements to deliver the backbone infrastructure that will be used to + capture the opportunities of digital technology and data to enable + transformation. + + + Figure 4. Urban Platform Development Stack. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 8 + + + Table 1. Urban Platform Development Stack + Layer Rationale + Societal Needs - Accessible services and data necessary to solve social problems + and drive innovation; + - Parameters that influence user’s experience while interacting with + services (e.g. usability, feeling of security and trust); + Services and Business - Tailor-made data services which careful targets the needs of users + Models and businesses; + - New potential and cost-effective beneficial services that could be + rolled out across cities of different sizes; + - Use cases where data is used to deliver different forms of value. + City Data - Data architectural features (e.g. volume, variety, temporal factors + and sensitivity); + - Data licensing, policies and regulations to exploit data to full effect; + - Minimum metadata requirements; + - Data usability and reusability aspects of humans and machines. + Urban Platform - Holistic and interoperable solutions; + - Integrated approaches which ensures that services fit together and + that synergies can be exploited; + - Data management mechanisms to ensure data integrity and + compliance with data protection regulations + - Extension capabilities to accommodate additional functionality at + later stage at a fair and transparent cost. + + + +1.8 User Classes, Characteristics, and User Access +The users of the Urban Platform include end-users, such as the general Public, public and private +organisations; data providers; service providers; and the platform providers who will be +working with the providers of city data and services, and managing the content, defining policies +and regulations of the platform. A crucial feature of an urban platform is the provision of the +various access levels required by the different types of users. Particular uses need different +access levels to some data than the general public. Data publishers will require access to the +Urban Platform in order to ingest, administer, manage, preserve and access their resources. This +will require multiple levels of access to city data and its respective metadata. Table 2 provides a +description of each class of users. + + + Table 2. Actors + User Class Rationale + Platform - Maintains the ecosystem of data, services and users; + Provider - Defines standards, licenses and regulations and provides terms and conditions + for platform usage and the commercial exploitation of data and services; + - Decides who are allowed to join the value network of data and services + providers; + City Data - Publishes open and proprietary data into the platform; + Publisher - Manages and maintain resources in the platform accordingly to terms and + conditions. + Data - Deploys open and commercial data services into the platform (e.g. data + Services visualisation, data cleansing, data integration tools); + Provider - Manages and maintain resources in the platform accordingly to terms and + conditions. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 9 + + + City Data - Consumes open and proprietary data provided in the platform; + Consumer - Uses open and commercial data services provided in the platform; + - Provides feedback on data and services provision; + + +1.8.1 End-User Access + +City data consumers will need to access and use the city data residing in the Urban Platform. End- +users will be able to search metadata and full text within datasets (when available), and obtain city +data in open formats readily available to both humans and machines such as CSV, XML, JSON. +Some end-users may require different access rights to city data. The 2 major end-user groups that +have been identified are: + +• Open data users, including both national and international users (humans and machines). + Open access to some city data may be restricted by licensing terms (e.g. commercial data), + embargo periods, copyright, etc. + +• Private data users, which need to use the Urban Platform to obtain commercial city data. Data + access is available via data subscriptions or when purchase requirements and licenses are + waived by the data provider. + + +1.8.2 City Data Publisher Access + +A broad data provider level access is needed for stakeholders (humans and machines) working +with the urban platform and their respective data in it. Basically, data publishers will carry out the +following activities: +• Data publication access, available to publishers adding new data and metadata, checking the + quality of datasets, manipulating data, performing format conversions, defining data-access + level, tariff for consumption when applicable, and licences. + +• Data maintenance access, for publishers reviewing or editing appropriate data and metadata in + the urban platform. Data publishers can view data and add to or edit metadata without + changing the data itself. They should be provided with access to feedback from users to + investigate problem in their resources (e.g. missing data, inconsistent metadata), and statistical + information about how their resources are used by users. + + +1.8.3 Data Services Provider Access + +This is the second most restrictive access level providing rights to deploy services in the platform. +Basically, data service providers will carry out the following activities: + +• Data services deployment access, available to service providers adding new mechanisms or + integrating new applications, testing and validating integration, defining data-access level and + tariff for service usage. + +• Data services maintenance access, for services providers reviewing, extending or editing + applications in the urban platform. Data services providers can view their services deployed + and add to or edit access level and tariff without having to deploy the services again. They + should be provided with access to feedback from users to investigate problem in their services + (e.g. bugs, scalability issues), and statistical information about how their services are used by + users. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 10 + + +1.8.4 Platform Provider Access + +This is the most restrictive access level providing ultimate rights to the system and is required for +its management, development, and assigning appropriate rights to data and services providers. +Policies and regulations, license agreements are also defined by the provider of the urban +platform. Platform providers should be provided with the means to follow up on civic engagement +(e.g. feedback, request for city data) and on the provision of city data and services. + + +1.9 User Documentation +• City Data Consumers: Provide license terms and conditions associated to consuming + data and services provided in the platform, documentation of APIs and guide to + discover city data in the platform. +• City Data Providers: Provide data publication documentation describing the minimum + metadata requirements, formats accepted, step-by-step guide to publish accurate city + data. +• Data Service Providers: Disclosure technical and architecture blueprint details in + order share and outsource expertise, and partnerships, and integrate supporting + partners’ solutions into the platform itself. + + +1.10 Design and Implementation Constraints +1.10.1 Design Constraints + +• Lack of standards agreement for metadata representation. diff --git a/files/core_requirements_part2.txt b/files/core_requirements_part2.txt new file mode 100644 index 0000000..fad6368 --- /dev/null +++ b/files/core_requirements_part2.txt @@ -0,0 +1,600 @@ +• City data found in existing data catalogues may require special consideration concerning the + type of formats and datasets that must be stored within the platform. +• Requirements mismatch due to increased number of stakeholders involved in the design + + +1.10.2 Implementation Constraints + +• Evaluation and testing of software options is expected to occur prior to selection and + implementation of a production urban platform. +• Budget costs are unknown until evaluation of software options is completed. + + +1.11 Assumptions, Alignment with other Action Clusters and Policies +1.11.1 Assumptions + +The assumptions in Table 3 have been identified by the Demand Side Engagement Stream as +relevant to this Requirements Specification. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 11 + + +Table 3. Assumptions + + # ASSUMPTION + + 1 The providers of city data and services will be responsible to maintain their resources in the platform. + + 2 All city data must meet the minimum metadata requirements and use the standards adopted by the platform. + + 3 The platform shall consider open Source as an optional commercial model, with open standards as a principle + + 4 The system design and architecture should minimize fragmentation of city data in the urban platform. + + 5 To the extent possible, automation should be used for the extraction of descriptive and technical metadata. + + 6 The platform must be designed in a way it accommodates additional functionality at later stage at a fair and transparent cost. + + The platform must be a modular based architecture which relies on stable and well-defined open interfaces to ensure + 7 + interoperability between the platform, services and the applications provided by service providers. + + The platform will offer open and well-documented API’s and clear service descriptions and contracts that is offered for reuse by + 8 another party to foster open innovation in the city, which means that developers and interested individuals openly utilize the + resources provided. + + 9 Adopt open and published European and International standards where possible. + + The platform must be flexible enough to accommodate different local, National and International data protection, licensing and +10 + commercialization regulations. + +11 Platform providers will monitor emerging technologies in order to maintain and improve the architecture. + + +12 Platform providers will monitor emerging information standards, including metadata standards and data interface standards. + + +13 Platform providers will monitor new commercial models for city data exploitation + + + + +1.11.2 Alignment with Citizen’s Focus Action Cluster +This specification document is aligned with the principles defined in the Citizen Focus5 Action +Cluster of the European Innovation Partnership on Smart Cities and Communities. Citizen Focus’ is +about “working together with citizens to realize public interests at the intersection of ICT, mobility +and energy in an urban environment”. We recognize citizens as owners of and participants in +the creation and delivery of city data and digital services, and we specify requirements to deliver +new digital services that will address the societal needs of cities in a positive manner that relates to +political narratives. Societal needs and wants are considered the starting point for city data service +offering by the urban platform. The requirements in this document were elicited considering + +• Human behaviour and needs as important as technology; +• The services and data that solve social problems and drive innovation; +• The mechanisms that make data and services more accessible to users; +• The factors that influences user’s experience while interacting with services provided (e.g. + usability, feeling of security and trust); + + +5 + https://eu-smartcities.eu/node/1333 + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 12 + + +1.11.3 Policies to be developed + +The following policies have been identified by the Demand Side Engagement Stream as relevant to +this Requirements Specification. + +• Data formatting and Metadata Schemes: Urban platforms will require more expansive, robust + and useful data encoding and conversion that what is available in existing data catalogues. + Data preservation policies should be developed to allow data to be stored in formats that can + be migrated, associated with metadata and ontologies to become both humans and machines + readable and understandable, ongoing monitoring for data obsolescence, and migrating data to + systems environments as needed to ensure their continued availability. Current Metadata + schemes (e.g. open data, sensory data ontologies) should be reviewed to see if it meets + current needs for city data management. It is possible that the needs of the urban platform + designed here may require new or additional schemas. + +• Data commercialisation: The commercial exploitation of city data and their funding models + are unexplored concepts that we are committed to address. There is an urgent need to define + license agreements and fair commercial and subscription models to allow interoperable open + and proprietary data to co-exist in the platform. + +• Data publication and services deployment: Policy development will be needed regarding + ingesting data and deploying data services into the platform, including which users/machines + will be authorized to submit data for publication, the minimum requirements for data submitted + by open and proprietary data publishers, and the removal of resources and services from the + platform. + + +2. Urban Platform Value Proposition, Use Cases and +Functional Requirements +2.1 From Value Proposition to Platform Specifications +This document uses goal-oriented modelling for eliciting, elaborating, structuring, specifying, +documenting, and modifying requirements. Goals represent the objectives which the urban +platform should achieve through cooperation of actors in the intended system and in the +environment. They capture, at different levels of abstraction, the various objectives the urban +platform under design should achieve. Through goals modelling we consider how the value +proposition and intended solutions connects across the stack, how the urban platform meets city +goals, why the system and its functionality are needed, and how the stakeholders’ interests may be +addressed. + +In our specification, we present the overall goal (the value proposition) that the urban platform +should aim to achieve in order to be considered as a viable final product, and a set of sub-goals +(intended solutions) it should maintain in the long run so that the overall goal can be unceasingly +achieved. By using this approach, the low-level technical requirements can be traced back to high- +level strategic objectives of the urban platform. The formal notations used in this document are: +Achieve [“Name of Overall Goal”] and Maintain [“Name of Sub-Goal”]. The requirements of the +Urban Platform are noted for each of the sub-goals, and are presented as a series of statements +regarding the capabilities needed in to achieve the overall goal of the Urban Platform. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 13 + + +2.2 Overall Goal: “City data is exploited to its full potential” +An urban platform is an organization of people and systems, which has accepted the responsibility +to preserve city data and make it available for all the stakeholders of smart cities. Ultimately, an +urban platform is a foundation for the full exploitation of city data. Hence, the major goal an Urban +Platform must achieve is “city data is exploited to its full potential”. To achieve this high level goal, +the urban platform must maintain in the long run the five sub-goals illustrated in Figure 3. + +Each one of the defined sub-goals co-enables the achievement of the specified high-level (overall) +goal of the urban platform. The sub-goals include the ingestion of city data, metadata generation, +data management, data storage, access, preservation, and administration, provision of engaging +services in the smart cities. These sub-goals are discussed in details in the following sections. + + + + Achieve [City data is exploited How we achieve the platform overall goal? + Why do we need the sub-goals? + to its full effect] + Goal + + + + + Maintain [User s experience is + Maintain [City data is provided + enhanced by the provision of + in a harmonised way] + S ub-Goal 1 S ub-Goal 5 value-added services] + co-ena bles + + + + + Maintain [Resources are managed in Maintain [City data is offered + a safe and intelligent manner] in an accessible manner] + S ub-Goal 2 S ub-Goal 4 + + + + + Maintain [City data is orchestrated + in a market place] + S ub-Goal 3 + + + + + Figure 3. Platform High-Level Goal and its respective sub-goals. + + + +2.2.1 Urban Platform Boundary + +The use case diagram illustrated in Figure 4 identifies the boundaries between the actors (either +automated or human) and the urban platform. We have arrived at the urban platform boundary by +inspecting each business use case and determining, in conjunction with the stakeholders needs, +which part of the business use case should be implemented and what part should be done by an +outsourced product (e.g. Billing System) using the framework 4. + +This task is technology agnostic and takes into account the abilities of the users/actors, the +constraints, the goals of the urban platform. Table 4 maps out the use cases with their respective +sub-goals and actors. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 14 + + + + + Manage Services + + S ub-Goal 5 + + + Data Service + Publish City Data + Provider + S ub-Goal 1 + + + <> Deploy Data Services + rules + S ub-Goal 5 + + Authenticate in the + Manage Resources <> + Platform Platform + S ub-Goal 1 Provider Database + <> Manage Infrastructure System + S ub-Goal 2 + + Utlise Data Services + City Data <> + Publisher S ub-Goal 5 + + Store City Data Management + <> + S ub-Goal 2 + Systems Services + System + + <> + City Data + Consumer + QoS Monitoring + Transmit Data + System + Register in the Platform S ub-Goal 2 + + + + + <> + rules + + <> + rules + Platform + Provider + rules + + Commercialise Data + Consume City Data + Services City Data + S ub-Goal 3 S ub-Goal 4 + Consumer + <> + + + Commercialise + City Data Discover City Data + S ub-Goal 3 + S ub-Goal 4 + + Billing + Management + System + + + + Authenticate in the Register in the + Platform Platform + + + + + Authenticate Authenticate + Register Consumers Register Publishers + Consumers Publishers + + + + + Figure 4. Simplistic overview of the use cases identified in the early stages of the platform design. + + + + +Table 4. Use Cases Mapping with Sub-Goals and Actors + Sub-Goals Use Cases ID Specialised Use Cases Actors +HIGH-LEVEL GOAL: City data is exploited to its + + + + + Publish City Data UC1 User publishes city data via data API’s + City Data Publisher + 1. City data is Register as a publisher + UC2 User manually uploads datasets + provided in a in the Platform + harmonised way + full effect + + + + + User manages resources + Manage Resources UC3 City Data Publisher + User tracks resources usage + + Store City Data UC4 + 2. City data is + managed in a safe Transmit Data UC5 - Management Systems + and intelligent + manner + Manage Infrastructure UC6 + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 15 + + + City Data Publisher + Set commercial city data + Platform Provider + Subscribe to proprietary data City Data Consumer + Commercialise City + UC7 + Data + Manage commercial data City Data Publisher + + 3. City data is Manage data subscription City Data Consumer + orchestrated in a + Data Services Provider + market place Set commercial data services + Platform Provider + Subscribe to commercial services Data Services Consumer + Commercialise Data + UC8 + Services + Manage commercial services Data Services Provider + + Manage services subscription Data Services Consumer + UC9 + Register in the Platform City Data Consumer + - + 4. City data is Discover City Data City Data Consumer + UC10 + offered in an + User consumes city data via data + accessible manner + API’s + Consume City Data UC11 City Data Consumer + User downloads datasets + 5. User’s Data Services Provider + Deploy Data Services UC12 + experience is Platform Provider + enhanced by the + Manage Services UC13 - Data Services Provider + provision of value- + added services + Utilise Data Services UC14 City Data Consumer + + + + + 2.3 SUB-GOAL 1: City data is collected in an intelligent manner + Description: The urban platform enables the owners of city data to easily publish both historic and + data streams in the platform, as well as their associated metadata. + Rationale: This sub-goal is maintained by the services and functions to accept the publication of + city data from data providers (of both open and proprietary data) and prepare the contents for + storage and management within the urban platform. Functions include receiving data, performing + quality assurance on data, verifying data formatting and document standards, associating meta- + data information, and coordinating updates to databases and resources management. + + Achieve [City data is exploited + to its full effect] + Goal + co-ena bles + + + + + How we achieve the platform overall goal? + + + + Maintain [City data is provided in + a harmonised manner] + S ub-Goal 1 + + + + +What actions can maintain the sub-goal? + + + + Achieve [Register Publisher] Achieve [Publish City Data] Achieve [Manage Resources] + UC1 UC2 UC3 + + + + + Figure 5. Sub-Goal 1 “City data is collected in an intelligent manner” refinement. + + + Drivers: Ensure data is published in an easy and uniform way. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 16 + + +Actions: For Sub-Goal 1 to be maintained in the long-run it requires the efficient realisation of use +cases: “Publish City Data” and “Manage Resources”, as shown in Figure 5. + + +2.3.1 Use Case: Register Publisher +ID: UC1 +Refines: SUB-GOAL 1: City data is collected in an intelligent manner +Pre-condition: Data Publisher is not logged in the system +Actors: Data Publishers +Rationale: Data Publishers can register in the platform and request approval to submit city data. +They provide valid registration details (to be defined) and wait for registration confirmation. +Platform Providers may authorise or not data publishers to offer both open and proprietary city data +in the platform. Data submission agreement is a formal agreement between the Data Provider and +the Urban Platform defining the terms of the content, standards, metadata creation, and license +agreement. The Urban Platform will proactively work with Data Providers to agree on the content, +quality and format of city data. Agreements between Platform and Data Providers may be +renegotiated on a periodic or ad-hoc basis. +Refines into requirements: FREQ.1 to FREQ.5. + + Use Case Basic Stimulus and Responses + 1. The platform prompts the user for a username and password or + register new account. + 2. The user selects registration option. + 3. The platform prompts user for publisher registration information (e.g. + username, password, organisation) + 4. The user enters in their information. + UC1. Register 5. Platform verifies information and creates account. + Publisher o If non-valid information, platform shows error message and + returns to step 1. + 6. Platform provider is requested to approve the account + o Platform acknowledges registration has been successful + o If non approved, platform shows error message and returns to + step 1. + 7. End of registration + + +2.3.2 Use Case: Publish City Data +ID: UC2 +Refines: SUB-GOAL 1 - City data is collected in an intelligent manner +Pre-condition: User is authenticated in the platform +Actors: City data publisher +Rationale: The Publish City Data function provides the appropriate mechanisms to receive city data from +authorized data providers. Data may be manually uploaded or submitted via APIs. In general, data providers +with whom the Urban Platform negotiates submission agreements are the providers of proprietary city data +(those producing published material, i.e. publishers) and open data, and they can be either humans or +machines. The providers of the Urban Platform will provide data providers with specifications on the content, +quality and format of data, and publication terms and conditions. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 17 + + +The Publish City Data function may represent a legal transfer of custody for the data in the urban platform, +and may require that special access controls be placed on the contents. This function provides a +confirmation of receipt of data publication to the Producer, which may include a request to resubmit data in +the case of errors resulting from the submission. +Once data has arrived, it must undergo several reviews, including virus checking, format compliance, +metadata minimum requirement agreement, quality and anticipated content and data formatting. The +platform must include the ability to record all actions and decisions made concerning the publication of city +data. The reasons for publication failure (e.g. missing metadata information, non-valid dataset) will be +provided back to the city data publisher. In some cases, the provider can then resubmit corrected data and +metadata information, while in other instances data publication refusal criteria should prevent the publisher +from submitting the same dataset at a later time period (e.g. in cases of suspicious datasets – copyrights +violation, viruses). When data is successfully submitted (either via APIs or manual upload), it will be +processed/prepared for storage into the platform’s database. +Specialised Use Cases: The Use Case Publish City Data data is distinguished into two +specialised Use Cases: “User publishes city data via data API’s (UC2.1)” and “User manually +uploads datasets (UC2.2)”. +Subordinated Use Cases: “Store City Data (UC4)” +Refines into requirements: FREQ 6 to FREQ.26. + + + Specialised Use Cases Basic Stimulus and Responses + 1. Platform provides user with an interface for static data + publication + 2. User selects datasets to be uploaded + 3. User provides metadata associated with the data (license, + provenance, ownership, semantics) in accordance with defined + standards + UC2.1. User manually 4. User requests data publication + uploads datasets 5. Platform quickly process user’s request for data publication + 6. Platform validates data submitted + o If valid data, platform acknowledges data publication has + been successful + o If non-valid data, platform shows error message and returns + to step 1. + 7. End of data publication + 1. Platform provides user with an interface for real-time data + publication + 2. User input data API information + 3. User provides metadata associated with the data (license, + provenance, ownership, semantics) in accordance with defined + standards + UC2.2. User publishes 4. User confirm information and request data publication + city data via data API’s 5. Platform quickly process user’s request for data publication + 6. Platform validates data submitted + o If valid data, platform acknowledges data publication has + been successful + o If non-valid data, platform shows error message and returns + to step 1 + End of data publication + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 18 + + +2.3.3 Use Case: Manage Resources +ID: UC3 +Refines: GOAL 1: City data is collected in an intelligent manner +Pre-condition: User successfully authenticates in the platform +Actors: City data publisher +Rationale: Manage resources provides the services and functions for updating, maintaining and +accessing both data and metadata, as well as tracking the usage of resources by users. Ideally +the owners of the resources should be the only authorised user to manage resources, and other +authorised users can track the usage of the resources in the platform. The platform must provide a +database update response indicating the status of the update, avoid update errors to be +propagated in the platform, and should keep an audit trail of all actions to enable rollback. Data +usage tracking includes performing queries on the data management data to generate result sets, +and producing reports from these result sets. +Specialised Use Cases: The Use Case Manage Resources data is distinguished into two +specialised Use Cases: “User manages resources (UC3.1)” and “User tracks resources usage +(UC3.2)”. +Subordinated Use Cases: “Transmit Data (UC5)” +Refines into requirements: FREQ.27 to FREQ.25. + + + Specialised Use Cases Basic Interactions and Responses + 1. Platform provides user with an interface for resources + management (e.g. data and metadata, data usage) + 2. User chooses to edit or delete data + 3. If edit, user revise metadata associated with the data (license, + provenance, ownership, access-control, semantics); + UC3.1. User manages 4. If delete, user selects dataset(s) to be removed + resources 5. User confirms action + 6. Platform quickly process user’s request + 7. Platform confirms execution of request + o If valid request, platform acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 8. End of resources management + 1. Platform provides user with an interface for resources + management (e.g. data and metadata, data usage) + 2. User chooses to visualise usage information of a dataset + UC3.2. User tracks 3. Platform quickly process user’s request for data usage + resources usage information + 4. Platform provides user with statistical information about data + usage and data users anonymised information + 5. End of data usage tracking. + + +2.3.4 Functional Requirements + + Req. ID UC. ID Description Priority Domain + Societal Needs, + FREQ.1 UC1 Allow data publishers to register to submit data for publication Must + Platform + Tracks data publication agreements between Data and Platform Business Needs, + FREQ.2 UC1 Must + Providers Platform + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 19 + + + Store terms of agreements, and use them to monitor/review/process + FREQ.3 UC1 Must City Data, Platform + data submissions. + Able to add and edit terms of agreement, based on access of level of Business Needs, + FREQ.4 UC1 Must + user. Platform + + FREQ.5 UC1 Data publications are managed and monitored Must City Data, Platform + + Allow authenticated users from across different organisations to City Data, Platform, + FREQ.6 UC2 Must + publish city data Business Needs + Provide authorization mechanisms for users and sensors to publish + FREQ.7 UC2 Must City Data, Platform + city data + City Data, Platform, + FREQ.9 UC2 Provide mechanisms for static data publication Must + Business Needs + City Data, Platform, + FREQ.10 UC2 Provide mechanisms for real-time data publication Must + Business Needs + + FREQ.11 UC2 Enable the publication of metadata Must City Data, Platform + + + FREQ.12 UC2 Maintain temporal information about the data Must City Data, Platform + + + FREQ.13 UC2 Support sensory data collection Must City Data, Platform + + + FREQ.14 UC2 Accepts content in numerous file types/formats Must City Data, Platform + + Prompts a request for resubmission to the data provider if an error of diff --git a/files/core_requirements_part3.txt b/files/core_requirements_part3.txt new file mode 100644 index 0000000..5e39fe9 --- /dev/null +++ b/files/core_requirements_part3.txt @@ -0,0 +1,600 @@ + harmonised way + full effect + + + + + User manages resources + Manage Resources UC3 City Data Publisher + User tracks resources usage + + Store City Data UC4 + 2. City data is + managed in a safe Transmit Data UC5 - Management Systems + and intelligent + manner + Manage Infrastructure UC6 + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 15 + + + City Data Publisher + Set commercial city data + Platform Provider + Subscribe to proprietary data City Data Consumer + Commercialise City + UC7 + Data + Manage commercial data City Data Publisher + + 3. City data is Manage data subscription City Data Consumer + orchestrated in a + Data Services Provider + market place Set commercial data services + Platform Provider + Subscribe to commercial services Data Services Consumer + Commercialise Data + UC8 + Services + Manage commercial services Data Services Provider + + Manage services subscription Data Services Consumer + UC9 + Register in the Platform City Data Consumer + - + 4. City data is Discover City Data City Data Consumer + UC10 + offered in an + User consumes city data via data + accessible manner + API’s + Consume City Data UC11 City Data Consumer + User downloads datasets + 5. User’s Data Services Provider + Deploy Data Services UC12 + experience is Platform Provider + enhanced by the + Manage Services UC13 - Data Services Provider + provision of value- + added services + Utilise Data Services UC14 City Data Consumer + + + + + 2.3 SUB-GOAL 1: City data is collected in an intelligent manner + Description: The urban platform enables the owners of city data to easily publish both historic and + data streams in the platform, as well as their associated metadata. + Rationale: This sub-goal is maintained by the services and functions to accept the publication of + city data from data providers (of both open and proprietary data) and prepare the contents for + storage and management within the urban platform. Functions include receiving data, performing + quality assurance on data, verifying data formatting and document standards, associating meta- + data information, and coordinating updates to databases and resources management. + + Achieve [City data is exploited + to its full effect] + Goal + co-ena bles + + + + + How we achieve the platform overall goal? + + + + Maintain [City data is provided in + a harmonised manner] + S ub-Goal 1 + + + + +What actions can maintain the sub-goal? + + + + Achieve [Register Publisher] Achieve [Publish City Data] Achieve [Manage Resources] + UC1 UC2 UC3 + + + + + Figure 5. Sub-Goal 1 “City data is collected in an intelligent manner” refinement. + + + Drivers: Ensure data is published in an easy and uniform way. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 16 + + +Actions: For Sub-Goal 1 to be maintained in the long-run it requires the efficient realisation of use +cases: “Publish City Data” and “Manage Resources”, as shown in Figure 5. + + +2.3.1 Use Case: Register Publisher +ID: UC1 +Refines: SUB-GOAL 1: City data is collected in an intelligent manner +Pre-condition: Data Publisher is not logged in the system +Actors: Data Publishers +Rationale: Data Publishers can register in the platform and request approval to submit city data. +They provide valid registration details (to be defined) and wait for registration confirmation. +Platform Providers may authorise or not data publishers to offer both open and proprietary city data +in the platform. Data submission agreement is a formal agreement between the Data Provider and +the Urban Platform defining the terms of the content, standards, metadata creation, and license +agreement. The Urban Platform will proactively work with Data Providers to agree on the content, +quality and format of city data. Agreements between Platform and Data Providers may be +renegotiated on a periodic or ad-hoc basis. +Refines into requirements: FREQ.1 to FREQ.5. + + Use Case Basic Stimulus and Responses + 1. The platform prompts the user for a username and password or + register new account. + 2. The user selects registration option. + 3. The platform prompts user for publisher registration information (e.g. + username, password, organisation) + 4. The user enters in their information. + UC1. Register 5. Platform verifies information and creates account. + Publisher o If non-valid information, platform shows error message and + returns to step 1. + 6. Platform provider is requested to approve the account + o Platform acknowledges registration has been successful + o If non approved, platform shows error message and returns to + step 1. + 7. End of registration + + +2.3.2 Use Case: Publish City Data +ID: UC2 +Refines: SUB-GOAL 1 - City data is collected in an intelligent manner +Pre-condition: User is authenticated in the platform +Actors: City data publisher +Rationale: The Publish City Data function provides the appropriate mechanisms to receive city data from +authorized data providers. Data may be manually uploaded or submitted via APIs. In general, data providers +with whom the Urban Platform negotiates submission agreements are the providers of proprietary city data +(those producing published material, i.e. publishers) and open data, and they can be either humans or +machines. The providers of the Urban Platform will provide data providers with specifications on the content, +quality and format of data, and publication terms and conditions. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 17 + + +The Publish City Data function may represent a legal transfer of custody for the data in the urban platform, +and may require that special access controls be placed on the contents. This function provides a +confirmation of receipt of data publication to the Producer, which may include a request to resubmit data in +the case of errors resulting from the submission. +Once data has arrived, it must undergo several reviews, including virus checking, format compliance, +metadata minimum requirement agreement, quality and anticipated content and data formatting. The +platform must include the ability to record all actions and decisions made concerning the publication of city +data. The reasons for publication failure (e.g. missing metadata information, non-valid dataset) will be +provided back to the city data publisher. In some cases, the provider can then resubmit corrected data and +metadata information, while in other instances data publication refusal criteria should prevent the publisher +from submitting the same dataset at a later time period (e.g. in cases of suspicious datasets – copyrights +violation, viruses). When data is successfully submitted (either via APIs or manual upload), it will be +processed/prepared for storage into the platform’s database. +Specialised Use Cases: The Use Case Publish City Data data is distinguished into two +specialised Use Cases: “User publishes city data via data API’s (UC2.1)” and “User manually +uploads datasets (UC2.2)”. +Subordinated Use Cases: “Store City Data (UC4)” +Refines into requirements: FREQ 6 to FREQ.26. + + + Specialised Use Cases Basic Stimulus and Responses + 1. Platform provides user with an interface for static data + publication + 2. User selects datasets to be uploaded + 3. User provides metadata associated with the data (license, + provenance, ownership, semantics) in accordance with defined + standards + UC2.1. User manually 4. User requests data publication + uploads datasets 5. Platform quickly process user’s request for data publication + 6. Platform validates data submitted + o If valid data, platform acknowledges data publication has + been successful + o If non-valid data, platform shows error message and returns + to step 1. + 7. End of data publication + 1. Platform provides user with an interface for real-time data + publication + 2. User input data API information + 3. User provides metadata associated with the data (license, + provenance, ownership, semantics) in accordance with defined + standards + UC2.2. User publishes 4. User confirm information and request data publication + city data via data API’s 5. Platform quickly process user’s request for data publication + 6. Platform validates data submitted + o If valid data, platform acknowledges data publication has + been successful + o If non-valid data, platform shows error message and returns + to step 1 + End of data publication + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 18 + + +2.3.3 Use Case: Manage Resources +ID: UC3 +Refines: GOAL 1: City data is collected in an intelligent manner +Pre-condition: User successfully authenticates in the platform +Actors: City data publisher +Rationale: Manage resources provides the services and functions for updating, maintaining and +accessing both data and metadata, as well as tracking the usage of resources by users. Ideally +the owners of the resources should be the only authorised user to manage resources, and other +authorised users can track the usage of the resources in the platform. The platform must provide a +database update response indicating the status of the update, avoid update errors to be +propagated in the platform, and should keep an audit trail of all actions to enable rollback. Data +usage tracking includes performing queries on the data management data to generate result sets, +and producing reports from these result sets. +Specialised Use Cases: The Use Case Manage Resources data is distinguished into two +specialised Use Cases: “User manages resources (UC3.1)” and “User tracks resources usage +(UC3.2)”. +Subordinated Use Cases: “Transmit Data (UC5)” +Refines into requirements: FREQ.27 to FREQ.25. + + + Specialised Use Cases Basic Interactions and Responses + 1. Platform provides user with an interface for resources + management (e.g. data and metadata, data usage) + 2. User chooses to edit or delete data + 3. If edit, user revise metadata associated with the data (license, + provenance, ownership, access-control, semantics); + UC3.1. User manages 4. If delete, user selects dataset(s) to be removed + resources 5. User confirms action + 6. Platform quickly process user’s request + 7. Platform confirms execution of request + o If valid request, platform acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 8. End of resources management + 1. Platform provides user with an interface for resources + management (e.g. data and metadata, data usage) + 2. User chooses to visualise usage information of a dataset + UC3.2. User tracks 3. Platform quickly process user’s request for data usage + resources usage information + 4. Platform provides user with statistical information about data + usage and data users anonymised information + 5. End of data usage tracking. + + +2.3.4 Functional Requirements + + Req. ID UC. ID Description Priority Domain + Societal Needs, + FREQ.1 UC1 Allow data publishers to register to submit data for publication Must + Platform + Tracks data publication agreements between Data and Platform Business Needs, + FREQ.2 UC1 Must + Providers Platform + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 19 + + + Store terms of agreements, and use them to monitor/review/process + FREQ.3 UC1 Must City Data, Platform + data submissions. + Able to add and edit terms of agreement, based on access of level of Business Needs, + FREQ.4 UC1 Must + user. Platform + + FREQ.5 UC1 Data publications are managed and monitored Must City Data, Platform + + Allow authenticated users from across different organisations to City Data, Platform, + FREQ.6 UC2 Must + publish city data Business Needs + Provide authorization mechanisms for users and sensors to publish + FREQ.7 UC2 Must City Data, Platform + city data + City Data, Platform, + FREQ.9 UC2 Provide mechanisms for static data publication Must + Business Needs + City Data, Platform, + FREQ.10 UC2 Provide mechanisms for real-time data publication Must + Business Needs + + FREQ.11 UC2 Enable the publication of metadata Must City Data, Platform + + + FREQ.12 UC2 Maintain temporal information about the data Must City Data, Platform + + + FREQ.13 UC2 Support sensory data collection Must City Data, Platform + + + FREQ.14 UC2 Accepts content in numerous file types/formats Must City Data, Platform + + Prompts a request for resubmission to the data provider if an error of + FREQ.15 UC2 Must City Data, Platform + data transmission or receipt occurs + + FREQ.16 UC2 Enable the semantic description of connected devices Must City Data, Platform + + + FREQ.17 UC2 Gather data from authenticated and authorized devices Must City Data, Platform + + + FREQ.18 UC2 Validates automatically the successful transfer of the data Must City Data, Platform + + + FREQ.19 UC2 Performs virus checking on data Must City Data, Platform + + Verifies the validity of the submission based on submitter, expected + FREQ.20 UC2 Must City Data, Platform + format, data quality, and completeness + Platform should have built-in checks on the incoming metadata. Data + FREQ.21 UC2 not containing the minimally defined set of attributes should be Must City Data, Platform + returned to the publisher for metadata enhancement. + System should have a user-friendly method of mapping non-standard + FREQ.22 UC2 Should City Data, Platform + metadata elements into approved standard elements. + Once ingested, metadata should be stored in a single common format. + FREQ.23 UC2 This format should be one that ensures against data loss, and allows a Must City Data, Platform + variety of access/distribution options + Data in the repository shall have sufficient technical metadata to + FREQ.24 UC2 assure functionality (e.g. viewing and display) to ensure accessibility Must City Data, Platform + and reusability. + Allows publishers to display and perform manual/visual quality control Business Needs, City + FREQ.25 UC2 Could + assurance via a user-friendly GUI Data, Platform + Business Needs, City + FREQ.26 UC2 Any errors shall prompt a request for resubmission of data Should + Data, Platform + + FREQ.27 UC3 Enable data providers to manage their resources Must Business Needs + + A minimal set of identifying information/metadata concerning data Business Needs, + FREQ.28 UC3 Must + publication submission must be recorded Platform + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 20 + + + Stores and tracks versions of data. Links /connections between + FREQ.29 UC3 Must City Data, Platform + versions are created and maintained + Give service and data providers access to anonymized data of the Business Needs + FREQ.30 UC3 Should + subscribers of their data or services + City Data, Platform, + FREQ.31 UC3 Enable data providers to maintain and repair data and metadata Should + Business Needs + Tracks data publication agreements between Data and Platform Business Needs, + FREQ.32 UC3 Must + Providers Platform + Store terms of agreements, and use them to monitor/review/process + FREQ.33 UC3 Must City Data, Platform + data submissions. + Able to add and edit terms of agreement, based on access of level of Business Needs, + FREQ.34 UC3 Must + user. Platform + + FREQ.35 UC3 Submission volumes and schedules are managed and monitored Must City Data, Platform + + + + +2.4 SUB-GOAL 2: City data is managed in a safe and intelligent manner +Rationale: The urban platform enables users to publish, consume and commercialise data, as well +as deploy and manage services all in a secure and privacy protected manner. + + Achieve [City data is exploited + to its full effect] + Goal + co-ena bles + + + + + Maintain [Resources are managed in a + safe and intelligent manner] + S ub-Goal 1 + + + + + Achieve [Store Data] Achieve [Transmit Data] Achieve [Manage Infrastructure] + UC4 UC5 UC6 + + + Figure 6. Sub-Goal 2 “City data is managed in a safe and intelligent manner” refinement. + + + +Drivers: Ensure data is secured and the identity of users are preserved +Actions: For Sub-Goal 2 to be maintained in the long-run it requires the efficient realisation of use +cases: “Store City Data” and “Retrieve and Transmit City Data”, as shown in Figure 6. + + +2.4.1 Use Case: Store City Data +ID: UC4 +Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner +Pre-condition: Data is successfully published in the platform +Actors: Urban Platform +Rationale: When data is successfully submitted (either via APIs or manual upload), it is +processed/prepared for storage into the platform’s database. This procedure will include the + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 21 + + +generation of unique identifiers to the database, enrichment with ontologies (when applicable), +encrypted (when applicable), signed with digital certificates (when applicable) to ensure that the +data conforms to the platform data formatting, standards, security and regulation. Data may be +converted to accepted formats, as needed (e.g. graph model). A primary goal of the conversion of +content for the platform is the preservation of the content. Priority will be given to preserving the +data accordingly to the policies defined in section (2.6.2). Access-control levels and license models +are associated to data which is subject to restrictions relating to access and conditions of use. +Refines into requirements: FREQ 28 – FREQ 39. + + Use Case Basic Interactions and Responses + 1. Platform mechanisms converts submitted data into a standard format + 2. Security enforcement (e.g. anonymisation, cryptography) is placed on + sensitive information. + UC4. Store City 3. Platform associates with datasets the access-control definitions set by + Data owner of resources. + 4. Data is enriched with semantics and is associated with other datasets + 5. Platform stores data in a scalable and secure database. + 6. End of data storage. + + +2.4.2 Use Case: Transmit Data +ID: UC5 +Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner +Pre-condition: Data is successfully published in the platform +Actors: Urban Platform +Rationale: The platform accepts data retrieval request, validates user’s rights to access the data, +retrieves city data from data storage, and moves a copy of the data to the relevant platform +component for further processing. If special processing is required, the retrieval function accesses +data in staging storage and applies the requested processes. The types of operations, which may +be carried out, include sub-sampling in temporal or spatial dimensions, conversions between +different data types or output formats, and other specialized processing (e.g., data visualisation). +Once it is finalised data will be sent to the appropriate delivery channels (e.g. API’s, GUI). It also +encompasses function to verify corruption during any internal data transfer. This function requires +that all hardware and software within the platform provide notification of potential errors and that +these errors are routed to standard error logs that are checked by the Platform Provider. +Refines into requirements: FREQ 40 to 54. +2.4.3 Use Case: Manage Infrastructure +ID: UC6 +Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner +Pre-condition: The platform is available +Actors: Urban Platform +Rationale: Manage infrastructure provides the services and functions for the overall operation of +the urban platform. Administration functions include monitoring quality of service agreements, +auditing data publication to ensure that they meet archive standards, and maintaining configuration +management of system hardware and software. In overall, it provides system engineering +functions to monitor and improve platform operations, and to inventory, report on, and +migrate/update the contents of the platforms’ databases. +Refines into requirements: FREQ 55 to 62. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 22 + + + + Use Case Basic Interactions and Responses + 1. Platform keeps monitoring services at run-time to ensure operation + and integrity of city data + o If system failure, the platform activates mechanisms for recovery + UC6. Manage based on pre-defined rules + Infrastructure o Platform logs issue and issue alert messages to platform providers + 2. Platform logs operation capabilities (e.g. performance, mean of time + failure, issues, etc.) + + +2.4.4 Functional Requirements + + Req. ID UC. ID Description Priority Domain + A minimal set of identifying information/metadata concerning data Business Needs, + FREQ.28 UC4 Must + publication submission must be recorded. Platform + Stores and tracks versions of data. Links /connections between + FREQ.29 UC4 Must City Data, Platform + versions are created and maintained. + + FREQ.30 UC4 Converts data to accepted file formats Must City Data, Platform + + Keep sensitive information secured and accessible only to authorized + FREQ.31 UC4 Must City Data, Platform + users + + FREQ.32 UC4 Keep user’s personal information protected Should City Data, Platform + + + FREQ.33 UC4 Keep city data and meta-data secured Must Platform Needs + + + FREQ.34 UC4 Enable privacy preserving mechanisms associated to data Must Platform Needs + + + FREQ.35 UC4 Model data in accordance with defined standards Must City Data, Platform + + + FREQ.36 UC4 Support the use of ontologies and semantic modelling of city data Could City Data + + + FREQ.37 UC4 Support database-level provenance annotation Should City Data + + + FREQ.38 UC4 Support data-level provenance annotation Should City Data + + + FREQ.39 UC4 Enable data to be encrypted Should Platform Needs + + System must have the ability to search and display metadata, preferably + City Data, Platform + FREQ.40 UC5 in a user-conformable, human readable display as well as in its native Must + Needs + format for machine harvesting and manipulation. + Controls access to data in the repository based on multiple permission + City Data, Platform + FREQ.41 UC5 levels. These permission levels determine the create/edit/read/delete Should + Needs + privileges granted users. + Access rights and conditions of use will be held for each data and its City Data, Platform + FREQ.42 UC5 Should + related metadata. Needs + Access rights and conditions can be inherited from a parent data to any City Data, Platform + FREQ.43 UC5 Could + data designated as a child data (derived information). Needs + Access rights and conditions of use will be machine readable and City Data, Platform + FREQ.44 UC5 Should + actionable. Needs + Access mechanisms must be sufficiently granular to allow the Business Needs, City + FREQ.45 UC5 identification of individual users, in order to maintain audit logs of actions Should + Data, Platform + performed by users. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 23 + + + Maintains the integrity of the database which contains both metadata + FREQ.46 UC5 and system information. Must Platform Needs + + Provides internal validation such as referential integrity of the contents of City Data, Platform + FREQ.47 UC5 the database. Must + Needs + Creates and maintains schema definitions required to support data + FREQ.48 UC5 management functions. Must Platform Needs + + Monitors and ensures that data and metadata are not corrupted during + FREQ.49 UC5 transfers. Must Platform Needs + + Provides statistically acceptable assurance that no components of the + FREQ.50 UC5 data are corrupted during any internal data transfer. Must Platform Needs + + Performs routine and special data integrity checking for each dataset + FREQ.51 UC5 and generates error reports. Must Platform Needs + + Provides disaster recovery capabilities including data backup, off-site + FREQ.52 UC5 data storage, data recovery, etc. Must Platform Needs + + Refresh/replace data without service interruption, and update + FREQ.53 UC5 corresponding metadata as appropriate. Must Platform Needs + + Ensure that any associated unique identifiers of the updated data are not + FREQ.54 UC5 altered. Must Platform Needs + + Audits submissions to ensure that they meet archive/repository + FREQ.55 UC6 standards. Must Platform Needs + + Maintains configuration management of the system hardware and + FREQ.56 UC6 software. Must Platform Needs + + Has capability to inventory, report on and migrate the contents of the + FREQ.57 UC6 repository. Must Platform Needs + + + FREQ.58 UC6 Ensures data integrity for version upgrades and format migration. Must Platform Needs + + + FREQ.59 UC6 Monitors functionality of the entire repository. Must Platform Needs + + + FREQ.60 UC6 Maintains integrity of system configuration. Must Platform Needs + + Audits system operations, performance and usage. + FREQ.61 UC6 Must Platform Needs + + Provides platform performance information and database holdings + FREQ.62 UC6 inventory reports Must Platform Needs + + + + +2.5 SUB-GOAL 3: City data is orchestrated in a marketplace +Rationale: The urban platform enables users to consume and publish data in a secure and privacy +protected manner. +Drivers: Ensure data is secured and the identity of users are preserved +Actions: For Sub-Goal 3 to be maintained in the long-run it requires the efficient realisation of use +cases: “Commercialise City Data” and “Commercialise Data Services”, as shown in Figure 7. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 24 + + diff --git a/files/core_requirements_part4.txt b/files/core_requirements_part4.txt new file mode 100644 index 0000000..b4cd2f1 --- /dev/null +++ b/files/core_requirements_part4.txt @@ -0,0 +1,700 @@ + city data via data API’s 5. Platform quickly process user’s request for data publication + 6. Platform validates data submitted + o If valid data, platform acknowledges data publication has + been successful + o If non-valid data, platform shows error message and returns + to step 1 + End of data publication + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 18 + + +2.3.3 Use Case: Manage Resources +ID: UC3 +Refines: GOAL 1: City data is collected in an intelligent manner +Pre-condition: User successfully authenticates in the platform +Actors: City data publisher +Rationale: Manage resources provides the services and functions for updating, maintaining and +accessing both data and metadata, as well as tracking the usage of resources by users. Ideally +the owners of the resources should be the only authorised user to manage resources, and other +authorised users can track the usage of the resources in the platform. The platform must provide a +database update response indicating the status of the update, avoid update errors to be +propagated in the platform, and should keep an audit trail of all actions to enable rollback. Data +usage tracking includes performing queries on the data management data to generate result sets, +and producing reports from these result sets. +Specialised Use Cases: The Use Case Manage Resources data is distinguished into two +specialised Use Cases: “User manages resources (UC3.1)” and “User tracks resources usage +(UC3.2)”. +Subordinated Use Cases: “Transmit Data (UC5)” +Refines into requirements: FREQ.27 to FREQ.25. + + + Specialised Use Cases Basic Interactions and Responses + 1. Platform provides user with an interface for resources + management (e.g. data and metadata, data usage) + 2. User chooses to edit or delete data + 3. If edit, user revise metadata associated with the data (license, + provenance, ownership, access-control, semantics); + UC3.1. User manages 4. If delete, user selects dataset(s) to be removed + resources 5. User confirms action + 6. Platform quickly process user’s request + 7. Platform confirms execution of request + o If valid request, platform acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 8. End of resources management + 1. Platform provides user with an interface for resources + management (e.g. data and metadata, data usage) + 2. User chooses to visualise usage information of a dataset + UC3.2. User tracks 3. Platform quickly process user’s request for data usage + resources usage information + 4. Platform provides user with statistical information about data + usage and data users anonymised information + 5. End of data usage tracking. + + +2.3.4 Functional Requirements + + Req. ID UC. ID Description Priority Domain + Societal Needs, + FREQ.1 UC1 Allow data publishers to register to submit data for publication Must + Platform + Tracks data publication agreements between Data and Platform Business Needs, + FREQ.2 UC1 Must + Providers Platform + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 19 + + + Store terms of agreements, and use them to monitor/review/process + FREQ.3 UC1 Must City Data, Platform + data submissions. + Able to add and edit terms of agreement, based on access of level of Business Needs, + FREQ.4 UC1 Must + user. Platform + + FREQ.5 UC1 Data publications are managed and monitored Must City Data, Platform + + Allow authenticated users from across different organisations to City Data, Platform, + FREQ.6 UC2 Must + publish city data Business Needs + Provide authorization mechanisms for users and sensors to publish + FREQ.7 UC2 Must City Data, Platform + city data + City Data, Platform, + FREQ.9 UC2 Provide mechanisms for static data publication Must + Business Needs + City Data, Platform, + FREQ.10 UC2 Provide mechanisms for real-time data publication Must + Business Needs + + FREQ.11 UC2 Enable the publication of metadata Must City Data, Platform + + + FREQ.12 UC2 Maintain temporal information about the data Must City Data, Platform + + + FREQ.13 UC2 Support sensory data collection Must City Data, Platform + + + FREQ.14 UC2 Accepts content in numerous file types/formats Must City Data, Platform + + Prompts a request for resubmission to the data provider if an error of + FREQ.15 UC2 Must City Data, Platform + data transmission or receipt occurs + + FREQ.16 UC2 Enable the semantic description of connected devices Must City Data, Platform + + + FREQ.17 UC2 Gather data from authenticated and authorized devices Must City Data, Platform + + + FREQ.18 UC2 Validates automatically the successful transfer of the data Must City Data, Platform + + + FREQ.19 UC2 Performs virus checking on data Must City Data, Platform + + Verifies the validity of the submission based on submitter, expected + FREQ.20 UC2 Must City Data, Platform + format, data quality, and completeness + Platform should have built-in checks on the incoming metadata. Data + FREQ.21 UC2 not containing the minimally defined set of attributes should be Must City Data, Platform + returned to the publisher for metadata enhancement. + System should have a user-friendly method of mapping non-standard + FREQ.22 UC2 Should City Data, Platform + metadata elements into approved standard elements. + Once ingested, metadata should be stored in a single common format. + FREQ.23 UC2 This format should be one that ensures against data loss, and allows a Must City Data, Platform + variety of access/distribution options + Data in the repository shall have sufficient technical metadata to + FREQ.24 UC2 assure functionality (e.g. viewing and display) to ensure accessibility Must City Data, Platform + and reusability. + Allows publishers to display and perform manual/visual quality control Business Needs, City + FREQ.25 UC2 Could + assurance via a user-friendly GUI Data, Platform + Business Needs, City + FREQ.26 UC2 Any errors shall prompt a request for resubmission of data Should + Data, Platform + + FREQ.27 UC3 Enable data providers to manage their resources Must Business Needs + + A minimal set of identifying information/metadata concerning data Business Needs, + FREQ.28 UC3 Must + publication submission must be recorded Platform + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 20 + + + Stores and tracks versions of data. Links /connections between + FREQ.29 UC3 Must City Data, Platform + versions are created and maintained + Give service and data providers access to anonymized data of the Business Needs + FREQ.30 UC3 Should + subscribers of their data or services + City Data, Platform, + FREQ.31 UC3 Enable data providers to maintain and repair data and metadata Should + Business Needs + Tracks data publication agreements between Data and Platform Business Needs, + FREQ.32 UC3 Must + Providers Platform + Store terms of agreements, and use them to monitor/review/process + FREQ.33 UC3 Must City Data, Platform + data submissions. + Able to add and edit terms of agreement, based on access of level of Business Needs, + FREQ.34 UC3 Must + user. Platform + + FREQ.35 UC3 Submission volumes and schedules are managed and monitored Must City Data, Platform + + + + +2.4 SUB-GOAL 2: City data is managed in a safe and intelligent manner +Rationale: The urban platform enables users to publish, consume and commercialise data, as well +as deploy and manage services all in a secure and privacy protected manner. + + Achieve [City data is exploited + to its full effect] + Goal + co-ena bles + + + + + Maintain [Resources are managed in a + safe and intelligent manner] + S ub-Goal 1 + + + + + Achieve [Store Data] Achieve [Transmit Data] Achieve [Manage Infrastructure] + UC4 UC5 UC6 + + + Figure 6. Sub-Goal 2 “City data is managed in a safe and intelligent manner” refinement. + + + +Drivers: Ensure data is secured and the identity of users are preserved +Actions: For Sub-Goal 2 to be maintained in the long-run it requires the efficient realisation of use +cases: “Store City Data” and “Retrieve and Transmit City Data”, as shown in Figure 6. + + +2.4.1 Use Case: Store City Data +ID: UC4 +Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner +Pre-condition: Data is successfully published in the platform +Actors: Urban Platform +Rationale: When data is successfully submitted (either via APIs or manual upload), it is +processed/prepared for storage into the platform’s database. This procedure will include the + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 21 + + +generation of unique identifiers to the database, enrichment with ontologies (when applicable), +encrypted (when applicable), signed with digital certificates (when applicable) to ensure that the +data conforms to the platform data formatting, standards, security and regulation. Data may be +converted to accepted formats, as needed (e.g. graph model). A primary goal of the conversion of +content for the platform is the preservation of the content. Priority will be given to preserving the +data accordingly to the policies defined in section (2.6.2). Access-control levels and license models +are associated to data which is subject to restrictions relating to access and conditions of use. +Refines into requirements: FREQ 28 – FREQ 39. + + Use Case Basic Interactions and Responses + 1. Platform mechanisms converts submitted data into a standard format + 2. Security enforcement (e.g. anonymisation, cryptography) is placed on + sensitive information. + UC4. Store City 3. Platform associates with datasets the access-control definitions set by + Data owner of resources. + 4. Data is enriched with semantics and is associated with other datasets + 5. Platform stores data in a scalable and secure database. + 6. End of data storage. + + +2.4.2 Use Case: Transmit Data +ID: UC5 +Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner +Pre-condition: Data is successfully published in the platform +Actors: Urban Platform +Rationale: The platform accepts data retrieval request, validates user’s rights to access the data, +retrieves city data from data storage, and moves a copy of the data to the relevant platform +component for further processing. If special processing is required, the retrieval function accesses +data in staging storage and applies the requested processes. The types of operations, which may +be carried out, include sub-sampling in temporal or spatial dimensions, conversions between +different data types or output formats, and other specialized processing (e.g., data visualisation). +Once it is finalised data will be sent to the appropriate delivery channels (e.g. API’s, GUI). It also +encompasses function to verify corruption during any internal data transfer. This function requires +that all hardware and software within the platform provide notification of potential errors and that +these errors are routed to standard error logs that are checked by the Platform Provider. +Refines into requirements: FREQ 40 to 54. +2.4.3 Use Case: Manage Infrastructure +ID: UC6 +Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner +Pre-condition: The platform is available +Actors: Urban Platform +Rationale: Manage infrastructure provides the services and functions for the overall operation of +the urban platform. Administration functions include monitoring quality of service agreements, +auditing data publication to ensure that they meet archive standards, and maintaining configuration +management of system hardware and software. In overall, it provides system engineering +functions to monitor and improve platform operations, and to inventory, report on, and +migrate/update the contents of the platforms’ databases. +Refines into requirements: FREQ 55 to 62. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 22 + + + + Use Case Basic Interactions and Responses + 1. Platform keeps monitoring services at run-time to ensure operation + and integrity of city data + o If system failure, the platform activates mechanisms for recovery + UC6. Manage based on pre-defined rules + Infrastructure o Platform logs issue and issue alert messages to platform providers + 2. Platform logs operation capabilities (e.g. performance, mean of time + failure, issues, etc.) + + +2.4.4 Functional Requirements + + Req. ID UC. ID Description Priority Domain + A minimal set of identifying information/metadata concerning data Business Needs, + FREQ.28 UC4 Must + publication submission must be recorded. Platform + Stores and tracks versions of data. Links /connections between + FREQ.29 UC4 Must City Data, Platform + versions are created and maintained. + + FREQ.30 UC4 Converts data to accepted file formats Must City Data, Platform + + Keep sensitive information secured and accessible only to authorized + FREQ.31 UC4 Must City Data, Platform + users + + FREQ.32 UC4 Keep user’s personal information protected Should City Data, Platform + + + FREQ.33 UC4 Keep city data and meta-data secured Must Platform Needs + + + FREQ.34 UC4 Enable privacy preserving mechanisms associated to data Must Platform Needs + + + FREQ.35 UC4 Model data in accordance with defined standards Must City Data, Platform + + + FREQ.36 UC4 Support the use of ontologies and semantic modelling of city data Could City Data + + + FREQ.37 UC4 Support database-level provenance annotation Should City Data + + + FREQ.38 UC4 Support data-level provenance annotation Should City Data + + + FREQ.39 UC4 Enable data to be encrypted Should Platform Needs + + System must have the ability to search and display metadata, preferably + City Data, Platform + FREQ.40 UC5 in a user-conformable, human readable display as well as in its native Must + Needs + format for machine harvesting and manipulation. + Controls access to data in the repository based on multiple permission + City Data, Platform + FREQ.41 UC5 levels. These permission levels determine the create/edit/read/delete Should + Needs + privileges granted users. + Access rights and conditions of use will be held for each data and its City Data, Platform + FREQ.42 UC5 Should + related metadata. Needs + Access rights and conditions can be inherited from a parent data to any City Data, Platform + FREQ.43 UC5 Could + data designated as a child data (derived information). Needs + Access rights and conditions of use will be machine readable and City Data, Platform + FREQ.44 UC5 Should + actionable. Needs + Access mechanisms must be sufficiently granular to allow the Business Needs, City + FREQ.45 UC5 identification of individual users, in order to maintain audit logs of actions Should + Data, Platform + performed by users. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 23 + + + Maintains the integrity of the database which contains both metadata + FREQ.46 UC5 and system information. Must Platform Needs + + Provides internal validation such as referential integrity of the contents of City Data, Platform + FREQ.47 UC5 the database. Must + Needs + Creates and maintains schema definitions required to support data + FREQ.48 UC5 management functions. Must Platform Needs + + Monitors and ensures that data and metadata are not corrupted during + FREQ.49 UC5 transfers. Must Platform Needs + + Provides statistically acceptable assurance that no components of the + FREQ.50 UC5 data are corrupted during any internal data transfer. Must Platform Needs + + Performs routine and special data integrity checking for each dataset + FREQ.51 UC5 and generates error reports. Must Platform Needs + + Provides disaster recovery capabilities including data backup, off-site + FREQ.52 UC5 data storage, data recovery, etc. Must Platform Needs + + Refresh/replace data without service interruption, and update + FREQ.53 UC5 corresponding metadata as appropriate. Must Platform Needs + + Ensure that any associated unique identifiers of the updated data are not + FREQ.54 UC5 altered. Must Platform Needs + + Audits submissions to ensure that they meet archive/repository + FREQ.55 UC6 standards. Must Platform Needs + + Maintains configuration management of the system hardware and + FREQ.56 UC6 software. Must Platform Needs + + Has capability to inventory, report on and migrate the contents of the + FREQ.57 UC6 repository. Must Platform Needs + + + FREQ.58 UC6 Ensures data integrity for version upgrades and format migration. Must Platform Needs + + + FREQ.59 UC6 Monitors functionality of the entire repository. Must Platform Needs + + + FREQ.60 UC6 Maintains integrity of system configuration. Must Platform Needs + + Audits system operations, performance and usage. + FREQ.61 UC6 Must Platform Needs + + Provides platform performance information and database holdings + FREQ.62 UC6 inventory reports Must Platform Needs + + + + +2.5 SUB-GOAL 3: City data is orchestrated in a marketplace +Rationale: The urban platform enables users to consume and publish data in a secure and privacy +protected manner. +Drivers: Ensure data is secured and the identity of users are preserved +Actions: For Sub-Goal 3 to be maintained in the long-run it requires the efficient realisation of use +cases: “Commercialise City Data” and “Commercialise Data Services”, as shown in Figure 7. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 24 + + + + Achieve [City data is exploited + to its full effect] + Goal + + + + + co-ena bles + Maintain [City data is orchestrated in + a marketplace] + S ub-Goal 3 + + + + + Achieve [Commercialise Data + Achieve [Commercialise City Data] + Services] + UC7 UC8 + + + + + Figure 7. Sub-Goal 3 “City data is orchestrated in a marketplace” refinement. + + + +2.5.1 Use Case: Commercialise City Data +ID: UC7 +Refines: SUB-GOAL 3: City data is orchestrated in a marketplace +Pre-condition: Data is successfully published in the platform, both publisher and consumers of +city data are authenticated in the platform, and there are billing capabilities available. +Actors: Data Publishers, Data Consumers +Rationale: The providers of city data can commercialise city data based on the policies and +financial models defined in the platform. After publishing their data, publishers can define which +data can be available as open data and which data should be available with the payment of a +subscription fee. Once publishers define which data is to be commercially exploited, the platform +will associate the data with their respective financial models and let it ready for subscription. City +data consumer chooses which data to purchase and is redirected to a billing interface where the +subscription payment is taken. The platform must provide an update response indicating the status +of the payment. If successful, data is ready available to be consumed by humans and machines, +otherwise the user can re-try the payment or cancel transaction. +Commercialise city data also involves the function of managing commercial data. It provides +services and functions for updating, maintaining and accessing both data and its respective +commercial transactions. Furthermore, it enables data providers to track the usage of commercial +data by users. Ideally the owners of the data should be the only authorised user to manage +resources, and other authorised users can track the usage of the data in the platform. Data usage +tracking includes performing queries on the data management data to generate result sets, and +producing reports from these result sets. Data consumers are provided with functions which enable +them to manage their subscriptions and financial transactions on the platform. These functions +include updating, maintaining and accessing financial transactions. For all these functions and +services, the platform must provide a database update response indicating the status of the +update, avoid update errors to be propagated in the platform, and should keep an audit trail of all +actions to enable rollback. +Specialised Use Cases: The Use Case Commercialise City Data is distinguished into four +specialised Use Cases: “Commercialise Data (UC7.1)”, “Consume Commercial City Data (UC7.2)”, +“Manage Commercial Data (UC7.3)” and “Manage Data Subscription (UC7.4)”. +Refines into requirements: FREQ 63 to 68. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 25 + + + + + Use Case Basic Interactions and Responses + 1. User selects the data to be commercialised + 2. User selects the commercial model for data consumption + UC7.1. 3. Platform validates selection + Commercialise 4. Platform associates data to subscription model + Data 5. Platform releases data for commercial exploitation in the marketplace + 6. End of data commercialisation set up. + 1. User selects the data to be subscribed to + 2. User request subscription to data + UC7.2. 3. Platform validates selection + Consume 4. Platform redirects user to billing system + Proprietary 5. Billing system deals with user request + Data o If successful, user is redirected to a GUI where data is ready to use + o If unsuccessful, user can try payment again or cancel request + 2. End of data subscription. + 1. Platform provides user with an interface for commercial data + management + 2. User chooses to edit or delete commercial data + o If edit, user revise commercial models, licenses, access-control, + semantics; + o If delete, user selects dataset(s) to be removed (following policies + UC7.3. Manage defined in the platform for data removal) + commercial 3. User confirms action + Data 4. Platform promptly process user’s request + 5. Platform confirms execution of request + o If valid request, platform acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 6. End of resources management + 1. Platform provides user with an interface for data subscription + management + 2. User chooses to edit or cancel data subscription + o If edit, user revise payment and subscription timeframe; + o If cancel, user selects dataset(s) to have subscription cancelled + UC7.4. Manage (following policies defined in the platform for data subscription) + data 3. User confirms action + subscription 4. Platform promptly process user’s request + 5. Platform confirms execution of request + o If valid request, platform updates acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 6. End of data subscription management + + + +2.5.2 Use Case: Commercialise Data Services +ID: UC8 +Refines: SUB-GOAL 3: City data is orchestrated in a marketplace +Pre-condition: Data is successfully published in the platform, both publisher and consumers of +city data are authenticated in the platform, and there are billing capabilities available. +Actors: Data Service Providers, Data Publishers and Consumers + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 26 + + +Rationale: After deploying data services in the platform, the providers of data services can choose +to commercialise services based on the policies and financial models defined in the platform. Once +data service providers define which service(s) is (are) to be commercially exploited, the platform +will associate the services with their respective financial models and let available in the platform +applications module ready for use. User (either publisher or consumer) chooses which data +services to use, and in case a charged service is selected the platform redirects the user to a +billing interface where the payment is taken. The platform must provide an update response +indicating the status of the payment. If successful, service is ready available to be used, otherwise +the user can re-try the payment or cancel transaction. Note that data service owners should be +able to waive the payment of tariff to certain users’ categories. +Data services providers are offered with functions to manage their commercial services. The +platform provides functions for updating, maintaining and accessing both service and its respective +commercial transactions. Furthermore, it enables data services providers to track the usage of +services by users. Services usage tracking includes performing queries on the platform to +generate result sets, and producing reports from these result sets. The consumers of data services +are provided with functions which enable them to manage their subscriptions and financial +transactions. These functions include updating, maintaining and accessing financial transactions. +For all these functions and services, the platform must provide a database update response +indicating the status of the update, avoid update errors to be propagated in the platform, and +should keep an audit trail of all actions to enable rollback. +Specialised Use Cases: The Use Case Commercialise Data Services is distinguished into four +specialised Use Cases: “Commercialise Data Services (UC8.1)”, “Consume Data Services +(UC8.2)”, “Manage Commercial Services (UC8.3)” and “Manage Services Subscription (UC8.4)”. +Refines into requirements: FREQ 68 to 73. + + Use Case Basic Interactions and Responses + 1. User selects the data services to be commercialised + 2. User selects the commercial model for data service usage based on the + category of users + UC8.1. 3. Platform validates selection + Commercialise 4. Platform associates data services to subscription model + Data Services 5. Platform enables data service to be commercially exploited in the + marketplace + 6. End of services commercialisation set up. + 1. User selects the data service to be subscribed to + 2. User request subscription to service + 3. Platform validates selection + UC8.2. 4. Platform validates user’s category its respective commercial models + Consume Data 4. If applicable, platform redirects user to billing system + Services 5. Billing system deals with user’s request + o If successful, user is redirected to a GUI where data is ready to use + o If unsuccessful, user can try payment again or cancel request + 6. End of data subscription. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 27 + + + 1. Platform provides user with an interface for commercial data + management + 2. User chooses to edit or delete commercial data + o If edit, user revise commercial models, licenses, access-control, + semantics; + o If delete, user selects dataset(s) to be removed (following policies + UC8.3. Manage defined in the platform for data removal) + commercial 3. User confirms action + services 4. Platform promptly process user’s request + 5. Platform confirms execution of request + o If valid request, platform acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 6. End of resources management + 1. Platform provides user with an interface for data subscription + management + 2. User chooses to edit or cancel data subscription + o If edit, user revise payment and subscription timeframe; + o If cancel, user selects dataset(s) to have subscription cancelled + UC8.4. Manage (following policies defined in the platform for data subscription) + services 3. User confirms action + subscription 4. Platform promptly process user’s request + 5. Platform confirms execution of request + o If valid request, platform updates acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 6. End of data subscription management + + +2.5.3 Functional Requirements + + Req. ID UC. ID Description Priority Domain + City Data, Platform, + FREQ.63 UC7 Support the commercialization of city data Should Business Needs + City Data, Platform, + FREQ.64 UC7 Enable users to subscribe to city data through the payment of a tariff Should Business Needs + City Data, Platform, + FREQ.65 UC7 Enable users to manage their data subscriptions Should Business + Provide platform providers mechanisms to define the terms and + FREQ.66 UC7 Must Platform Needs + conditions for platform data usage + Enable data providers to manage the subscription models of their City Data, Platform, + FREQ.67 UC7 Should + data Business + UC7/ City Data, Platform, + FREQ.68 Utilise secure and reliable billing and payment management systems Must + UC8 Business + City Data, Platform, + FREQ.69 UC8 Support the commercialization of data services Should Business Needs + Enable data providers to manage the commercial models of their City Data, Platform, + FREQ.70 UC8 Should + services Business Needs + Provide service providers mechanisms to define the terms and City Data, Platform, + FREQ.71 UC8 Should + conditions of platform services Business Needs + Allow users to pay a tariff for using certain advanced services (e.g. City Data, Platform, + FREQ.72 UC8 Should + Data manipulation, enrichment) Business Needs + City Data, Platform, + FREQ.73 UC8 Enable users to manage their data services subscriptions Should Business Needs + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 28 + + +2.6 SUB-GOAL 4: City data is offered in an accessible manner +Rationale: The urban platform provides city data in both human and machine (e.g. sensors, +actuators, systems) readable and understandable formats. +Drivers: Ensure data understandability and machine-to-machine data transaction. +Actions: For Sub-Goal 3 to be maintained in the long-run it requires the efficient realisation of use +cases: “Register Consumer”, “Discover City Data”, and “Consume City Data” as shown in Figure 8. + + Achieve [City data is exploited + to its full effect] + Goal + + + + + co-ena bles + Maintain [City data is offered in an + accessible manner] + S ub-Goal 1 + + + + + Achieve [Register Consumer] Achieve [Discover City Data] Achieve [Consume City Data] + UC9 UC10 UC11 + + + + + Figure 8. Sub-Goal 4 “City data is offered in an accessible manner” refinement. + + + +2.6.1 Use Case: Register Data Consumer +ID: UC9 +Refines: SUB-GOAL 4: City data is offered in an accessible manner +Pre-condition: User is not logged in the platform +Actors: Data Consumers +Rationale: Data Consumers can register in the platform and request approval to consume city data +via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to +confirm their registration. Users must accept the terms and conditions of platform usage and define +how their personal data can be used by the Platform Owner. Users can manage and alter their +registration information at any time they want to. +Refines into requirements: FREQ 64. + + Use Case Basic Stimulus and Responses + 1. The platform prompts the user for a username and password or register + new account. + 2. The user selects registration option. + 3. The platform prompts user for data consumer registration information + UC1. Register 4. The user enters in their information. + Consumer 5. Platform verifies information and creates account. + o If non-valid information, platform shows error message and returns to + step 1. + 6. Platform acknowledges registration has been successful + 7. End of registration + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 29 + + +2.6.2 Use Case: Discover City Data +ID: UC10 +Refines: SUB-GOAL 4: City data is offered in an accessible manner +Pre-condition: User has access to either platform GUI or API +Actors: Data Consumers +Rationale: Data Consumers can register in the platform and request approval to consume city data +via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to +confirm their registration. +Refines into requirements: FREQ 64. + + Use Case Basic Stimulus and Responses + 1. Users access specialised data query end-points (e.g. SPARQL) + 2. Users provides information for pre-defined parameters for search + 3. Users request data search + 4. Platform quickly process users request for data diff --git a/files/core_requirements_part5.txt b/files/core_requirements_part5.txt new file mode 100644 index 0000000..69b68e6 --- /dev/null +++ b/files/core_requirements_part5.txt @@ -0,0 +1,604 @@ + Consume City o If authentication is successful, users are provided with requested + Data via APIs data streams + 3. Users are provided with requested data via APIs + + + +2.6.4 Functional Requirements + + Req. ID UC. ID Description Priority Domain + Allow users to register to use services and consume proprietary city Societal Needs, + FREQ.74 UC9 Should + data and open data (optional) Platform + Keep sensitive information secured and accessible only to Societal Needs, + FREQ.75 UC9 Should + authorized users Platform + Societal Needs, + FREQ.76 UC9 Provide authentication mechanisms for users Must Platform + Societal Needs, + FREQ.77 UC9 Keep user’s personal information protected Must Platform + Allow users to control which data they are willing to provide and how Societal Needs, + FREQ.78 UC9 Must + their data should be used Platform + Societal Needs, + FREQ.79 UC10 Allow users to format data in any supported data formats Must + Platform + The query request may require data to be sourced from different Societal Needs, + FREQ.80 UC10 Must + storage locations Platform + Allows query requests against all metadata used to manage the Societal Needs, + FREQ.81 UC10 repository. Should Platform + Provide users information about the legal aspects of the data Societal Needs, + FREQ.82 UC11 Must + (license, ownership) City Data + Societal Needs, + FREQ.83 UC11 Keeps an audit trail of all actions. Must City Data + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 31 + + +2.7 SUB-GOAL 5: User’s experience is enhanced by the provision of +value-added services +Rationale: The urban platform enables users to consume and publish data in a secure and privacy +protected manner +Drivers: Ensure data is secured and the identity of users are preserved +Actions: For Sub-Goal 5 to be maintained in the long-run it requires the efficient realisation of use +cases: “Deploy Data Services”, “Manage Services”, and “Utilise Data Services” as shown in Figure +8. + + Achieve [City data is exploited + to its full effect] + Goal + + + + + co-ena bles + Maintain [User s experience is enhanced by + the provision of value-added services] + S ub-Goal 1 + + + + + Achieve [Deploy Data Services] Achieve [Manage Services] Achieve [Utilise Data Services] + UC12 UC13 UC14 + + + + + Figure 9. Sub-Goal 5 “User’s experience is enhanced by the provision of value-added services” refinement + + + +2.7.1 Deploy Data Services +ID: UC12 +Refines: SUB-GOAL 5: User’s experience is enhanced by the provision of value-added services +Pre-condition: Data Services Providers are provided with credentials, are authorised to deploy +their services in the platform, and have access to technical specifications of the platform interfaces. +Actors: Data Services Providers +Rationale: Data Services can register in the platform and request approval to deploy services in +the platform. They provide valid registration details (to be defined) and wait for registration +confirmation. Platform Providers may authorise or not data services providers to offer both open +and proprietary services in the platform. Data services providers must formally agree with service +deployment agreement with the Urban Platform. This agreement defines terms of the content, +policies, regulations, license agreement. The Urban Platform will proactively work with Service +Providers to agree on the technical specifications of interfaces and platform openness level. +Agreements between Platform and Data Service Providers may be renegotiated on a periodic or +ad-hoc basis. +Refines into requirements: FREQ.1 to FREQ.5. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 32 + + + + Use Case Basic Stimulus and Responses + 1. User authenticates in the platform + 2. User is provided access to platform interfaces for service deployment + 3. User deploy services + 4. Platform checks compatibility and any technical issues arising from the + UC12. Deploy new service + Services 5. If approved the service is ready to be used and managed in the + platform + o If deployment is not approved, platform shows error message and + returns to step 1. + 6. End of deployment + + + +2.7.2 Use Case: Manage Services +ID: UC13 +Refines: GOAL 5: User’s experience is enhanced by the provision of value-added services +Pre-condition: User successfully authenticates in the platform +Actors: Data Services Providers +Rationale: Manage services provides the functions for updating, maintaining and accessing +services as well as tracking their usage by users. Ideally the owners of the services should be the +only authorised user to manage them, and other authorised users can track the usage of the +services in the platform. In case of updates the platform must log in the database update details +and send to service providers a response indicating the status of the update. The platform should +also ensure update errors are not propagated nor affect the health of other services provided in the +platform, and should keep an audit trail of all actions to enable rollback. Data services usage +tracking includes performing queries on the data management data to generate result sets, and +producing reports from these result sets. All user’s information provided to service providers must +follow regulations of data protection and the user’s defined rules for their data use. +Specialised Use Cases: The Use Case Manage Services is distinguished into two specialised +Use Cases: “User manages services (UC13.1)” and “User tracks service usage (UC13.2)”. +Subordinated Use Cases: “Transmit Data (UC5)” +Refines into requirements: FREQ.27 to FREQ.25. + + + Specialised Use Cases Basic Interactions and Responses + 1. Platform provides user with an interface for services management + 2. User chooses to edit or delete services + 3. If edit, user revise service information (access-control, + commercial models, parameters) and deployment; + If delete, user selects services to be removed / disabled + UC13.1. User manages 4. User confirms action + services 5. Platform quickly process user’s request + 6. Platform confirms execution of request + o If valid request, platform acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 7. End of services management + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 33 + + + 1. Platform provides user with an interface for services management + 2. User chooses to visualise usage information of a service + 3. Platform quickly process user’s request for data usage + UC13.2. User tracks information + services usage 4. Platform provides user with statistical information about services + usage and data users anonymised information + 5. End of data services tracking. + + + +2.7.3 Use Case: Utilise Data Services +ID: UC2 +Refines: SUB-GOAL 1 - City data is collected in an intelligent manner +Pre-condition: User is authenticated in the platform +Actors: City data publisher +Rationale: Data Consumers can register in the platform and request approval to consume city data via GUI +or APIs. They provide valid registration details (to be defined) and wait for platform to confirm their +registration. +Subordinated Use Cases: “Commercialise Data Services (UC8)” +Refines into requirements: FREQ 6 to FREQ.26. + + + Specialised Use Cases Basic Interactions and Responses + 1. Users / Machines select data service to be utilised + 2. Users / Machines are redirected to authentication mechanism in + UC2.1. User utilises data case of registration is needed for the particular service + services o If authentication is successful, users are provided with + requested data streams + 3. Users are provided with requested service either via API or GUI + + +2.7.4 Functional Requirements + + Req. ID UC. ID Description Priority Domain + Provide stable and well-defined interfaces to ensure interoperability + Societal Needs, + FREQ.84 UC12 between the platform, services and the applications provided by Should Platform + services providers + Ensure the interfaces of the architecture are open to reduce entry Societal Needs, + FREQ.85 UC12 Should + barriers and integration issues Platform + Provide multi-purposed and network intelligent interfaces to Societal Needs, + FREQ.86 UC12 Must + providers and consumers of services Platform + Provide service providers mechanisms to define the terms and Societal Needs, + FREQ.87 UC12 Must + conditions of platform services deployment Platform + Business Needs, + FREQ.88 UC13 Provide statistical information of user’s feedback on service usage Must Platform + Allow users to use services to manipulate city data (e.g. Create Societal Needs, + FREQ.89 UC10 Must + mash ups, integrate) Platform + Allow users to provide feedback on usability, and quality of data Societal Needs, + FREQ.90 UC14 Must + and services provided by the platform Platform + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 34 + + + +3. Non-functional Requirements +3.1 Run-time Quality Requirements +3.1.1 Scalability Requirements + +Rationale: The ability of the system to execute its task within its expected performance profile and +to handle on-demand increased processing volumes of data and service requests +Drivers: Provide ready access to all data that underpins decision making processes in smart cities, +and accommodate users and data usage patterns +Refines into requirements: NFREQ.1, NFREQ.2, NFREQ.3, NFREQ.4 +Relevance: Urban platforms have ambitious performance requirements. Such platforms must cope +with user’s demand for data and services, capture real-time data that will be catalysed by a myriad +of sensors. The demand for urban platform is very likely to significantly increase over time. It is +very difficult to have clear performance characteristics due to the ubiquity, heterogeneity high +connectivity of devices and end users. + + SCALABILITY MEASURES + Actions Capture the performance requirements + Create service level agreements + Predict scalability using software simulation + Analyse the performance of the platform overtime + Conduct practical testing + Strategy Prioritize service and data requests + Distribute processing over time + Scale up or scale out as necessary + Reuse resources and results Partition and parallelize + Constantly monitor Quality of Service at runtime + + + +3.1.2 Availability and Reliability Requirements + +Rationale: The ability of the system to be fully or partly operational as and when required and to +effectively handle failures that could affect system availability +Drivers: Build a reliable foundation for “on demand” exploitation of data +Refines into requirements: NFREQ.5, NFREQ.6, NFREQ.7, NFREQ.8 +Relevance: Any system that has complex or extended availability requirements, complex recovery +processes, or a high profile (e.g., is visible to the public) + + AVAILABILITY AND RELIABILITY MEASURES + Actions Capture the availability requirements Produce the availability schedule + Estimate platform availability Estimate functional availability Assess against the + requirements Rework the architecture + Strategy Adopt fault-tolerant hardware + Use reliable infrastructure database + Log transactions + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 35 + + + Develop adaptive software to cope and recover from faults + Design and test for failure + Deploy load balancing + Identify suitable backup and disaster recovery solution + + +3.1.3 Trust Requirements + +Rationale: A quality related to the user’s belief in the reliability, integrity and ability of the functional +behaviour of the platform +Drivers: Gain understanding of what influences user’s experience while interacting with services +provided +Requirements: NFREQ.16, NFREQ.17 +Relevance: Relevant to the systems that share and collect information that may raise public +concern. In some cases, trust has to do with the reliability of data and their providers, whereas in +other cases trust can be associated with the security and privacy of the technology that was +deployed. Trust affects the reputation of the platform besides its dissemination and maturity on the +market. + +TRUST MEASURES + Capture trust requirements + Perform risk analysis so measures can be implemented +Actions Explore the vulnerability aspects of city data + Check whether extensibility requirements impact on trust + Define a trust model + Adopt trust model + Deploy monitoring capabilities and assess its impact on scalability + Manage the data in a way that ensures its compliance with data +Strategy protection regulations + Implement tampering and data misuse detection + Make use of Cryptography when necessary + Allow Federation of trust between platforms + + + +3.1.4 Security Requirements + +Rationale: Ability of the system to enforce the intended confidentiality, trust, integrity and service +and data access policies, and to detect and recover from failure in these security mechanisms. +Drivers: Manage the data and services in a way that ensures its integrity, and compliance with +data protection regulations +Relevance: Relevant to the systems that share and collect information that may raise public +concern. Urban platforms may become a valuable target for attackers which can potentially leave +huge swathes of information exposed. It could potentially undermine trust in the government and +damage its reputation. +Refines into requirements: NFREQ.9 to NFREQ.15 + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 36 + + + SECURITY MEASURES + Elicit the security requirements + Cross-check security requirements’ impact on services offered by different + providers + Verify security impact on service composition + Conduct risk analysis and impacts of security breach + Use scalable and efficient user authentication components + Use authentication and authorization components to secure interfacing with + Actions external services + Address aspects of data collection, storage and distribution + Address aspects of service and communication security among devices + Identify security requirements of the physical infrastructure + Balance and prioritize scalability (performance) and security requirements + Evaluate trade-offs between privacy considerations and preventing abuse. (While + anonymous access guarantees privacy of users, traceability of users due to abuse + of the service is not possible.) + Toughen user’s functional components + Authenticate subjects + Define and enforce access policies + Secure communication infrastructures + Secure interfaces with external systems and services + Strategy + Secure databases + Check data integrity for critical services + User digital certificates and encryption where necessary + Secure monetary transactions + Secure user’s personal information + + +3.1.5 Privacy Requirements + +Rationale: Ability of the system to ensure that the collection and transmission of personal data is +minimized and handled in accordance with user’s expectation and regulations. +Drivers: Protect the vulnerability aspects volunteered citizen’s data +Requirements: NFREQ.18, NFREQ.19, NFREQ.20, NFREQ.21, NFREQ.22 +Relevance: Ensuring users privacy is protected positively influences user’s experience, +acceptance and continuous use of the platform. Besides other factors, the reputation of the +platform depends on how well user’s information is secure and preserved. + + PRIVACY MEASURES + Capture trust requirements + Perform risk analysis so measures can be implemented + Actions Explore the vulnerability aspects of city data + Check whether extensibility requirements impact on trust + Define a trust model + Allow users to interact with the platform anonymously in given + circumstances + Manage users identification securely + Strategy Anonymize data whenever necessary to comply with data regulations + Cryptograph personal identifiers during data transmission when that is + necessary + Avoid unauthorized access to implicit information (e.g. location) + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 37 + + + Verify the impact of security, trust and scalability requirements trade-offs + on privacy + Allow the user to control how personal information is used + Empower user to control the data disclosure mechanism + + + +3.2 Non Run-time Quality Requirements +3.2.1 Evolvability Requirements + +Rationale: The ability of the platform to withstand and easily adapt when new requirements and +changes is introduced. +Drivers: Ensure the platform is able to accommodate additional functionality and emerging +technologies at later stage at a fair and transparent cost +Refines into requirements: NFREQ.23 +Relevance: Important for longer- lived and more widely used systems. Urban platforms are +expected to be highly evolvable in order to accommodate future emerging technologies and avoid +interoperability issues. + + EVOLVABILITY MEASURES + Actions Characterize and assess the evolution needs + Consider the evolution trade-offs + Balance and negotiate potential conflicting requirements emerging from + the evolution capability. + Strategy Adopt standards and open interfaces + Design loose-coupled components + Preserve the platform resilience + + + +3.2.2 Extensibility Requirements + +Rationale: The flexibility of the system to allow services and functionality to be extended and +augmented by service providers in order to increase value of services to both platform providers +and end-users. The extension of the platform services is determined by the Platform Openness +strategy. +Drivers: Stakeholders can extend the services provided by the urban platform, so that +partnerships can be built to deliver holistic and interoperable solutions. Identify integrated +approaches to design and service delivery which ensures that services fit together and that +synergies can be exploited. +Refines into requirements: NFREQ.24, NFREQ.25 +Relevance: Important for longer- lived and more widely used systems. Urban platforms are +expected to be highly evolvable in order to accommodate future emerging technologies and avoid +interoperability issues. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 38 + + + EXTENSIBILITY + Actions Characterize and assess the extensibility needs + Trace the impacts and cost of extensions + Negotiate potential conflicting requirements emerging from adding extensions + in the platform + Strategy Develop a modular architecture with standard interfaces that reduces entry + barriers due to increased transparency and integration + Share technical information about interfaces will help service providers in + targeting opportunities around the platform + + + +3.3 List of Non-Functional Requirements + +ID # Description Concern Priority Domain + + Platform, +NFREQ.1 Support different service level agreements (SLA) Scalability Should + Infrastructure + Platform, +NFREQ.2 Process services and events on a set of distributed nodes Scalability Should + Infrastructure + Platform, +NFREQ.3 Continuously monitor Quality of Service at runtime Scalability Should + Infrastructure + Platform, +NFREQ.4 Balance its load at runtime Scalability Should + Infrastructure + Platform, +NFREQ.5 Provide high availability Availability Should + Infrastructure + Platform, +NFREQ.6 Guarantee infrastructure availability Availability Should + Infrastructure + Platform, +NFREQ.7 Ensure network availability Availability Should + Infrastructure + Platform, +NFREQ.8 Be able to perform self-healing Availability Should + Infrastructure + City Data, +NFREQ.9 Expose data and services to authorized users Security Must + Platform + City Data, +NFREQ.10 Ensure services are always accessible to entitled users Security Must + Platform + +NFREQ.11 Ensure Data Freshness Security Must Platform + +NFREQ.12 Support access control mechanisms Security Must Platform + + Platform, +NFREQ.13 Have security mechanisms to protect data transmission Security Should + Infrastructure + Platform, +NFREQ.14 Make it difficult to spy on communicated messages Security Should + Infrastructure + Platform, +NFREQ.15 Be able to perform to detect threats at runtime Security Should + Infrastructure + Provide trusted and secure communication and information Platform, +NFREQ.16 Trust Should + management Infrastructure + +NFREQ.17 The platform infrastructure and services shall be trustable Trust Should Infrastructure + + Societal Needs, +NFREQ.18 Allow users to use free services anonymously Privacy Should + Platform + Societal Needs, +NFREQ.19 Allow people to use free services anonymously Privacy Should + Platform + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 39 + + + Allow users to control which data they are willing to provide Societal Needs, +NFREQ.20 Privacy Should + and how their data should be used Platform + +NFREQ.21 Keep users access-control rights/ policies secured. Privacy Should Platform + + Provide privacy protection for users interacting with the Societal Needs, +NFREQ.22 Privacy Should + platform Platform + Platform, +NFREQ.23 Provide communication confidentiality Privacy Should + Infrastructure + Societal Needs, +NFREQ.24 Be extensible for future technologies. Evolvability Must City Data, Platform, + Infrastructure + Platform, +NFREQ.26 Provide standard interfaces for service providers Extensibility Should + Business Needs + +NFREQ.26 Be able to provide services in an interoperable manner Extensibility Should Platform + + + + +4. Other Requirements +4.1 Minimal Descriptive Metadata Required from Data Provider + +Minimal Descriptive Metadata Required from Data Provider + + Attribute Required Definition Notes Examples + +Database Name Y A name given to the resource + +Author Name of a person or body associated with + Y + the creation of the resource + +Maintainer Name of the entity responsible for making + Y + the resource available + +Date Created Y Date the dataset was created + + +Date Modified Y Date the dataset was modified + + +Last Revision Y Date the dataset was last revised + + +Update Frequency Y Frequency of data maintenance + + +Mode of Release Y Open/Proprietary + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 40 + + +4.2 Urban Platform Supported Data Formats + + +MIME type Description Extensions Level + +application/json JavaScript Object Notation json supported + +application/xml eXtensible markup language file xml supported + +text/csv Comma separated values file csv supported + + + + +5. Conclusion & Forward Plans +This document represents the first set of requirements specification for urban platform. Future +activities will collaboratively assess, resolve requirements conflicts, prioritize, and validate the +requirements of the urban platform. Ultimately, this document will become a complete final +requirements specification document to guide and speed up the development open platform for +cities. Table below shows the milestones, deliverables and engagement activities completed and +yet to be completed by the Demand Side Engagement Stream. + + + + Milestones, deliverables and engagement activity Forecast date Status + Stakeholders invitation Late Completed + September/2015 + Online Workshop 1: Project Scope and Outcomes 20/11/2015 Completed + Online meeting held with city members to communicate goals, + expected outcomes, time commitment and required actions. + Start of Collaborative Requirements Engineering Process 20/11/2015 Completed + Participants reviewed the first draft of urban platform requirements, + provided comments, and suggested new requirements and changes. + Online Workshop 2: Review and Refine Requirements 08/12/2015 Completed + Participants had a conference call to review the requirements + specification document and provide their comments. + Deliverable: Requirements Specification Document v2.1 shared 05/01/2016 Completed + across the Working Streams for consultation + Online Workshop 3: Review Requirements Specification 12/01/2016 Completed + Document and discussion of Letter of Intent + Participants had a conference call to review the requirements + specification document and provide their comments, and discussed + the Letter of Intent to be signed by Mayor/Equiv of EU cities to + commit to application of common U.P. approach. + Urban Platforms Workshop in Brussels 19/01/2016 Completed + Participants provided feedback on the requirements specification + document and recommended changes in the document + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 41 + + + Deliverable: Updated Requirements Specification Document v2.2 29/01/2016 Completed + shared with Industry and Standardisation Working Streams + Deliverable: Online release of the Requirements Specification Early + Document and open calls for wider consultation on the Requirements February/2016 + + Online Workshop 4: Engagement with Industry Side Early + Capture industry feedback on the Requirements Specification February/2016 + Document to assess its suitability to drive the development of the first + draft reference architecture. + Online Workshop 5: Review Requirements Specification Mid + Document v2.2 and discuss requirement prioritization and February/2016 + balancing + Collectively determine which candidate requirements of urban + platforms should be prioritized as high importance to all the cities + (use of Value Oriented Prioritization Method). Requirements are also + prioritized to minimize risk during development of urban platforms so + that the most important requirements are made explicit and + considered in the reference architecture. + Capture of case studies from cities intended to support requirements Mid March/2016 + validation (City of Porto), and learning and confidence building + amongst cities. diff --git a/files/extracted_requirements.txt b/files/extracted_requirements.txt new file mode 100644 index 0000000..80f9fc2 --- /dev/null +++ b/files/extracted_requirements.txt @@ -0,0 +1,2455 @@ +Software Requirements Specification for Urban Platforms (EIP Project) Page i + + + + + Requirements Specification + For + + Urban Platforms + Version 2.2 + (Consultation Stage) + + + + + Prepared by Demand-Side Engagement Stream + + + + +European Innovation Partnership for Smart Cities & Communities (EIP_SCC) +Integrated infrastructure Action Cluster – Urban Platform + Software Requirements Specification for Urban Platforms (EIP Project) Page ii + + + + +Requirements Specification Life-Cycle +Revision history + + Author Date Reason for change Phase Revision + + Dr. Larissa Romualdo-Suzuki 4/12/2015 - Initial Draft Version 0 + Outline + + Dr. Larissa Romualdo-Suzuki 21/12/2015 Working update Core Demand- Version 1 + Side Team + Consultation + + Dr. Larissa Romualdo-Suzuki 05/01/2016 Incorporate cities Development Version 2.1 + comments + + Dr. Larissa Romualdo-Suzuki 29/01/2016 Incorporate suggestions Development Version 2.2 + from Urban Platforms + Workshop held in Brussels + on the 19/01/16. + + + + +Requirements Specification sign-off: + + Name: Andrew Collinge + Role: Demand Side Engagement Lead Date + Organisation: Greater London Authority 11 January 2016 + + + + +Acknowledgement: + +This document has been developed in consultation with a representative group of European cities from +different locations, size, and level of development. Cities include: London, Amsterdam, Barcelona, +Syracuse, Berlin, Gent, Valencia, Murcia, Derry, Copenhagen, Scottish cities, Porto, Riga. We wish to +recognize the commitment, support and ongoing contribution from these cities to develop a common +statement of needs. + Software Requirements Specification for Urban Platforms (EIP Project) Page iii + + + + +Table of Contents +Table of Contents ...................................................................................................................... iii +1. Introduction ............................................................................................................................1 + 1.1 Purpose and Scope of this Document.................................................................................. 1 + 1.2 Context ..................................................................................................................................... 1 + 1.3 Intended Audience .................................................................................................................. 2 + 1.4 Definitions, Standards, and Framework ............................................................................... 3 + 1.5 How to Use this Document .................................................................................................... 2 + 1.6 Product Scope and Perspective ............................................................................................ 5 + 1.7 The Urban Platform Development Stack ............................................................................. 7 + 1.8 User Classes, Characteristics, and User Access ............................................................... 8 + 1.9 User Documentation ............................................................................................................. 10 + 1.10 Design and Implementation Constraints ............................................................................ 10 + 1.11 Assumptions, Alignment with other Action Clusters and Policies ................................... 10 +2. Urban Platform Value Proposition, Use Cases and Functional Requirements ..12 + 2.1 From Value Proposition to Platform Specifications .......................................................... 12 + 2.2 Overall Goal: “City data is exploited to its full potential” .................................................. 13 + 2.3 SUB-GOAL 1: City data is collected in an intelligent manner ......................................... 15 + 2.4 SUB-GOAL 2: City data is managed in a safe and intelligent manner .......................... 20 + 2.5 SUB-GOAL 3: City data is orchestrated in a marketplace ............................................... 23 + 2.6 SUB-GOAL 4: City data is offered in an accessible manner ........................................... 28 + 2.7 SUB-GOAL 5: User’s experience is enhanced by the provision of value-added + services............................................................................................................................................... 31 +3. Non-functional Requirements..........................................................................................34 + 3.1 Run-time Quality Requirements .......................................................................................... 34 + 3.2 Non Run-time Quality Requirements.................................................................................. 37 + 3.3 List of Non-Functional Requirements ................................................................................. 38 +4. Other Requirements ...........................................................................................................39 + 4.1 Minimal Descriptive Metadata Required from Data Provider .......................................... 39 + 4.2 Urban Platform Supported Data Formats .......................................................................... 40 +5. Conclusion & Forward Plans ...........................................................................................40 +Appendix ......................................................................................... Error! Bookmark not defined. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 1 + + + + +1. Introduction +1.1 Purpose and Scope of this Document +The purpose of this document is to provide a common core set of city-needs-led requirements, co- +developed and validated by EU cities, to support the acquisition of “Urban Platforms” by EU cities. +The requirements are made openly available. The ambition is that adoption of these requirements +by EU cities will lead to reduced pre-procurement times, increased confidence in platform designs, +greater levels of collaboration (particularly amongst smaller cities), innovation in business models, +more affordable solutions, and a more secure basis for industry to apply its innovations. This will +lead to accelerated adoption of urban platforms by EU cities, so that they can exploit the potential +of the growing volumes of city data, and improve the services to and outcomes for their residents, +business community and visitors. + +This document provides a primary input to the EIP_SCC Urban Platform Supply-Side and +Standardisation workstreams as a reference for the development of further technical and +operational materials. The requirements illustrate the purpose and complete declaration for the +development of system, as well as system constraints, interface and interactions with other +external applications. + +The scope of this document is limited to common services data platform requirements, and issues +related to the management of city data within an Urban Platform. This considers the full life-cycle +of city data, including: the maintenance of the inventory of existing city data; development of +functional requirements for common services data platform from a city-needs perspective; +identification of metadata and format standards for city data; and identification of policy issues +related to the data platform. It includes in the scope the functionality of software and systems used +to handle existing data catalogues (e.g., open data portals). + +Commercial off-the-shelf (COTS) software or hardware, as well as databases, which manage data +that are not part of Urban Platform’s collecting responsibility (e.g., mobile applications, proprietary +Internet of Things, proprietary data catalogues) are outside the scope of these requirements. + + +1.2 Context +The initial EIP_SCC Demand-Side survey1 clearly indicated that the vast majority of EU cities did +not have an urban platform; that there were a number of significant barriers to adoption (including +funding, capabilities, and the willingness to work across service ‘silos’); that there was limited +collaboration in this area; and as a result most cities were ‘sitting on the fence’. The opportunity +clearly is to unblock this. Notably because urban platforms provide a vital foundation for smart city +infrastructure and service improvements. So their adoption is considered a strategic priority by the +EIP_SCC, and by a growing number of cities. + +There are a number of clear trends that any city, and thus these requirements, must recognise: + • The astounding increase in volumes of city data, driven notably by IoT/sensor/M2M + implementations, and social media + • The pressure to improve and make open data from public (and private) sources; + +1 + Survey performed Q1’15 and reported on the EIP_SCC Marketplace site + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 2 + + + • The continued reality of austerity that drives cities towards transformative solutions + +Cities presently typically have a mix and variety of qualities of legacy systems, with various +degrees of technical ‘platform or platforms’ in place. It is recognised that the roadmap a city might +follow to transition to a more aligned and functional state will differ in many ways; and that the end +point does not envisage ‘one (physical) platform’. However cities will benefit from adopting more +consistent conceptual, and logical architectures, which will lead to more efficient and effective +physical operations, and greater benefits in terms of exploiting city data. + +Furthermore it must be recognised that cities will continue to wrestle with capacity challenges to +improve quality, availability, governance, and appropriate monetisation of city data. + + +1.3 Intended Audience +This document is co-developed by and initially intended for the growing number of city members of +the Demand-Side Engagement Stream of the EIP_SCC Urban Platform initiative. Its formal release +as a common statement of requirements by representative EU cities will form the basis for +commitment by EU cities to the initiative by way of a Letter of Intent (LoI). The spirit of which seeks +to ensure broad use of these requirements by EU cities (and recognition of deviations) to help +open the smart city market. + +It is also intended to be used by Industry in setting an agreed city-needs-led set of requirements +which can become more consistent and common across EU cities. This will lead to advantage for +cities and Industry. + + +1.4 How to Use this Document +Cities are encouraged to use this document as a guiding reference for the development of their +own platform specifications. + +The following steps offer a logical means to extract the full value of this document: + + + 1. Identify the priority service outcomes that the city seeks to achieve to steer the overall + prioritisation of urban platform development + 2. Perform a city data mapping exercise to develop a picture of the data landscape, including: + sources, volume, variety, temporal factors and sensitivity, data licensing and ownership + policies. + 3. Map out existing ICT system resources across the city in order to identify those resources + with the greatest potential for reuse, identify gaps and provide the foundation for a data + strategy and technology plan to fill them. + 4. Identify the requirements within this document that best suit the city needs - alongside any + other emerging requirements – and refine and prioritise them in accordance to their data + strategy defined in step 1 and 2. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 3 + + + 5. Cities can create their own specification document in their own language and refer to the + requirements of this document by using the unique identifier of the requirements (which + signals to providers the similarities and deviations from the core set) + 6. Determine what ICT infrastructure and sets of requirements are needed citywide to support + the urban platform. + + + + +1.5 Definitions, Standards, and Framework +To ensure a robust basis for market adoption we have set in place a number of foundations. Firstly, +a common definition of terms. Secondly, the adoption of some core accepted standards. Thirdly, +an underlying model to develop requirements. + +1.5.1 Working Definitions +For the purpose of this document we assume the working definitions developed within and shared +across all working groups of the Urban Platform initiative + +• An ‘Urban Platform’… + + o Implements a logical architecture/content/design that brings together (integrates) data flows + within and across city systems and exploits modern technologies (sensors, cloud services, + mobile devices, analytics, social media etc); + o Provides the building blocks that enable cities to rapidly shift from fragmented operations to + include predictive effective operations, and novel ways of engaging and serving city + stakeholders; + o Transforms, in a way that is tangible and measurable, outcomes at local level (e.g. increase + energy efficiency, reduce traffic congestion and emissions, create (digital) innovation + ecosystems, efficient city operations for administrations and services). + +• ‘City Data’ is… + + o Data that is held by any organisation - government, public sector, private sector or not-for- + profit - which is providing a service or utility, or is occupying part of the city in a way that can + be said to have a bearing on local populations and the functioning of that place; + o Consists of varied characteristics such as static, near-real time or in the future, real time, + descriptive or operational. + o It will be to a greater extent generated by individual citizens and this too (with due + consideration to privacy and a strong trust framework) can be considered city data. + + +For the purpose of this report, the following terms and definitions4 apply. + + o Open data: non-privacy-restricted and non-confidential data. Produced with either public or + private resource and is made available without any restrictions on its usage or distribution. + o Private data: restricted and/or licensed data including permission, charging, privacy, + publication and distribution. Produced with either public or private resource. + o Commercial data: restricted and/or licensed data including permission, charging, privacy, + publication and distribution. Produced with either public or private resource. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 4 + + + o Sensory data: open and/or restricted data collected by different sensors, actuators and + devices owned by public and private sector, and citizens. Sensory data is usually diverse in + nature and architectural features, mostly location and time dependent, and present different + levels of quality. + +These types of city data are produced within the public and private sectors and crowdsourcing +initiatives, and have their respective suppliers and distribution strategies (e.g. reports, APIs, +datasets). + + o Public sector data: restricted data relating to location, national security, commercial + sensitivity and privacy. Data produced, collected or funded by the public sector. + o Private sector data: data produced, collected or funded by the private sector, which can be + open, private and commercial data. In case of private data, the restrictions of usage and + distribution are decided by the individual businesses. + o Crowd-sourced data: data provided, collected and distributed by humans through the use + of digital technologies and social media. + + +1.5.2 Use of Standards + +• We seek to ensure Urban Platforms in EU cities are designed with regard to international best + practices for data repositories. +• As new guidance and standards are developed the three streams of the EIP_SCC2 Urban + Platform initiative will review them for broader use. (Note: the SCC03 EC project and + consortium is explicitly linked within this initiative and will act on Urban Platform standards). +• For now, this document assumes a modified IEEE 830-19983 layout for software requirements + specification, as the basis to capture a high-level statement of the urban platform’s + requirements. + + +1.5.3 The Organising Framework + +This document uses the dynamic business models framework4 illustrated in Figure 1 to organise +and guide the definition of goals, the development of use cases, and definition of requirements +during the life cycle of urban platforms. Future iterations of this dynamic design process are +Procurement, Roll-Out and Market phases. Consequently, this document may include, alter, +prioritize, extend or remove specifications as they become known. + +The principal goal addressed is that of “exploiting the value of city data through urban +platforms”. From this we have developed five common sub-goals. Each includes: a description; +rationale; drivers; and actions. These sub-goals are elaborated through a total of 14 use cases and +developed into a total of 116 functional and non-functional requirements as illustrated in Figure 2. +The use of the requirements by Industry will be subject to a further update. + + + + +2 + EIP Integrated infrastructure Urban Platform Governance Model, Version 3, 28th July 2015. +3 + IEEE 830-1998 Standard for Software Requirements Specifications, 1998. +4 + Framework for Data Infrastructures Design from “Data as Infrastructure for Smart Cities”, Suzuki-LCSR, +PhD Thesis, UCL, 2015. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 5 + + + + + Figure 1. Organising Framework4. + + + + + Sub-Goal 1: City data is provided + Use Cases: 3 Requirements: 34 + in a harmonised way + value of city data through an + Principle Goal: Exploit the + + + + + Sub-Goal 2: City data is managed + Use Cases: 3 Requirements: 36 + in a safe and intelligent manner + urban platform + + + + + Sub-Goal 3: City data is + Use Cases: 2 Requirements: 12 + orchestrated in a market place + + + + + Sub-Goal 4: City data is offered in + Use Cases: 3 Requirements: 11 + an accessible manner + + + + + Sub-Goal 5: User’s experience is + enhanced by the provision of Use Cases: 3 Requirements: 8 + value-added services + + + + Figure 2 Structure of this document. + + + + +1.6 Product Scope and Perspective +The EIP Project’s urban platform is an open common architecture which serves for city data +collection, management and distribution. An urban platform is intended to support the widespread +exploitation of city data by humans and machines in the urban environment. Figure 3 illustrates a +holistic high level overview of the urban platform the EIP project intends to deliver. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 6 + + + +The reference architecture for urban platforms should: + +• Cater for interoperability between urban infrastructures +• Enable replicability of the solutions/platforms city to city +• Scale without technical constraints and excessive cost increase +• Provide open APIs and SDKs +• Enable Real Time capabilities +• Support implementation of functional and technical capabilities + + + + + Figure 3. High level overview of the urban platform (currently approved EC DG CNECT). + + +The current urban platform market is nascent. Many software vendors offer such a platform, though +many requirements and expectations of the stakeholders of city data are not (fully) addressed. As +a result, current urban platforms are often more costly to design and maintain, less reusable and +often not interoperable platform-to-platform, and susceptible to information fragmentation and +overload. + +The urban platform which the EIP initiative intends to design takes a step beyond the platforms +currently on the market by ensuring the requirements are fully founded on a co-created and +common set of representative city needs, from which it solicits suitable industry input, and an open +and managed collaboration between industry, cities and communities, and others, in order to take +into account their needs and concerns. To do this, it is necessary to take a technology agnostic +approach to design an open and common reference architecture for urban platforms. This platform +must ensure data is collected and sustained in accordance with well-stablished standards, +managed in a robust manner so that it can handle high level supply and demand of data, and +distributed across different value chains, systems and stakeholders. The ability to handle high level +of city data supply and demand while being user secure and accessible enough for city-wide +exploitation of data is one of many keys to the success of urban platforms. This is central to the +design and implementation of urban platforms. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 7 + + +1.7 The Urban Platform Development Stack +The requirements specification of this document is based on the Urban Platform Development +Stack illustrated in Figure 4. The stack is composed by five domains (represented as layers in the +stack) necessary to fully implement an urban platform software suite as shown in Table 1. Each +domain comprehends a set of requirements necessary to the design of a common and open urban +data service platform. The elicited requirements are used to define a technical architecture which is +simple enough to be comprehensible at least at a high level of abstraction. The platform should be +conceptually decomposable into its major subsystems, the platform’s functionality reused by many +services and external applications should be identifiable, and interactions between the platform +and services, data providers and data consumers should be well defined and explicit. + +The first layer of the stack “Societal needs” concerns to outcomes we strive for within a portfolio of +city service domains. An urban platform should recognise societal needs and wants as the starting +point for city data service offering. Ultimately, an urban platform aims to provide tailor made and +compelling engaging services for the users. The Services and Business models layers concerns +with delivering data services which carefully targets the needs and expectations of the different +users of the urban platform, and explore use cases and commercial models where data is used to +deliver different forms of value. The city data layer concerns with the mechanisms necessary to +transform urban platforms into a foundation for widespread exploitation of data, including handling +data architectural features, data usability, semantics and quality aspects. The urban platform layer +concerns to the technology foundation to configure, share, and interpret exponentially increasing +volumes city data and services. Finally, the Infrastructure layer concerns with the base level +connectivity that supports the platform to be scalable and reliable in the long run. + + + + STACK OUTPUT + + Requirements to deliver new digital services that will address the + Societal Needs societal needs of cities in a positive manner that relates to political + narratives. + + + Services & Requirements to new profitable business models and the development + Business Models of an increase range of new and engaging services in the smart cities. + + + + City Data Requirements to provide all city data stakeholders ready access and + delivery of all city data that unpins the decision making process in smart + cities. + + + Urban Platform Requirements to put in place applications together to build a foundation + for the widespread exploitation of data. + + + Infrastructure Requirements to deliver the backbone infrastructure that will be used to + capture the opportunities of digital technology and data to enable + transformation. + + + Figure 4. Urban Platform Development Stack. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 8 + + + Table 1. Urban Platform Development Stack + Layer Rationale + Societal Needs - Accessible services and data necessary to solve social problems + and drive innovation; + - Parameters that influence user’s experience while interacting with + services (e.g. usability, feeling of security and trust); + Services and Business - Tailor-made data services which careful targets the needs of users + Models and businesses; + - New potential and cost-effective beneficial services that could be + rolled out across cities of different sizes; + - Use cases where data is used to deliver different forms of value. + City Data - Data architectural features (e.g. volume, variety, temporal factors + and sensitivity); + - Data licensing, policies and regulations to exploit data to full effect; + - Minimum metadata requirements; + - Data usability and reusability aspects of humans and machines. + Urban Platform - Holistic and interoperable solutions; + - Integrated approaches which ensures that services fit together and + that synergies can be exploited; + - Data management mechanisms to ensure data integrity and + compliance with data protection regulations + - Extension capabilities to accommodate additional functionality at + later stage at a fair and transparent cost. + + + +1.8 User Classes, Characteristics, and User Access +The users of the Urban Platform include end-users, such as the general Public, public and private +organisations; data providers; service providers; and the platform providers who will be +working with the providers of city data and services, and managing the content, defining policies +and regulations of the platform. A crucial feature of an urban platform is the provision of the +various access levels required by the different types of users. Particular uses need different +access levels to some data than the general public. Data publishers will require access to the +Urban Platform in order to ingest, administer, manage, preserve and access their resources. This +will require multiple levels of access to city data and its respective metadata. Table 2 provides a +description of each class of users. + + + Table 2. Actors + User Class Rationale + Platform - Maintains the ecosystem of data, services and users; + Provider - Defines standards, licenses and regulations and provides terms and conditions + for platform usage and the commercial exploitation of data and services; + - Decides who are allowed to join the value network of data and services + providers; + City Data - Publishes open and proprietary data into the platform; + Publisher - Manages and maintain resources in the platform accordingly to terms and + conditions. + Data - Deploys open and commercial data services into the platform (e.g. data + Services visualisation, data cleansing, data integration tools); + Provider - Manages and maintain resources in the platform accordingly to terms and + conditions. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 9 + + + City Data - Consumes open and proprietary data provided in the platform; + Consumer - Uses open and commercial data services provided in the platform; + - Provides feedback on data and services provision; + + +1.8.1 End-User Access + +City data consumers will need to access and use the city data residing in the Urban Platform. End- +users will be able to search metadata and full text within datasets (when available), and obtain city +data in open formats readily available to both humans and machines such as CSV, XML, JSON. +Some end-users may require different access rights to city data. The 2 major end-user groups that +have been identified are: + +• Open data users, including both national and international users (humans and machines). + Open access to some city data may be restricted by licensing terms (e.g. commercial data), + embargo periods, copyright, etc. + +• Private data users, which need to use the Urban Platform to obtain commercial city data. Data + access is available via data subscriptions or when purchase requirements and licenses are + waived by the data provider. + + +1.8.2 City Data Publisher Access + +A broad data provider level access is needed for stakeholders (humans and machines) working +with the urban platform and their respective data in it. Basically, data publishers will carry out the +following activities: +• Data publication access, available to publishers adding new data and metadata, checking the + quality of datasets, manipulating data, performing format conversions, defining data-access + level, tariff for consumption when applicable, and licences. + +• Data maintenance access, for publishers reviewing or editing appropriate data and metadata in + the urban platform. Data publishers can view data and add to or edit metadata without + changing the data itself. They should be provided with access to feedback from users to + investigate problem in their resources (e.g. missing data, inconsistent metadata), and statistical + information about how their resources are used by users. + + +1.8.3 Data Services Provider Access + +This is the second most restrictive access level providing rights to deploy services in the platform. +Basically, data service providers will carry out the following activities: + +• Data services deployment access, available to service providers adding new mechanisms or + integrating new applications, testing and validating integration, defining data-access level and + tariff for service usage. + +• Data services maintenance access, for services providers reviewing, extending or editing + applications in the urban platform. Data services providers can view their services deployed + and add to or edit access level and tariff without having to deploy the services again. They + should be provided with access to feedback from users to investigate problem in their services + (e.g. bugs, scalability issues), and statistical information about how their services are used by + users. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 10 + + +1.8.4 Platform Provider Access + +This is the most restrictive access level providing ultimate rights to the system and is required for +its management, development, and assigning appropriate rights to data and services providers. +Policies and regulations, license agreements are also defined by the provider of the urban +platform. Platform providers should be provided with the means to follow up on civic engagement +(e.g. feedback, request for city data) and on the provision of city data and services. + + +1.9 User Documentation +• City Data Consumers: Provide license terms and conditions associated to consuming + data and services provided in the platform, documentation of APIs and guide to + discover city data in the platform. +• City Data Providers: Provide data publication documentation describing the minimum + metadata requirements, formats accepted, step-by-step guide to publish accurate city + data. +• Data Service Providers: Disclosure technical and architecture blueprint details in + order share and outsource expertise, and partnerships, and integrate supporting + partners’ solutions into the platform itself. + + +1.10 Design and Implementation Constraints +1.10.1 Design Constraints + +• Lack of standards agreement for metadata representation. +• City data found in existing data catalogues may require special consideration concerning the + type of formats and datasets that must be stored within the platform. +• Requirements mismatch due to increased number of stakeholders involved in the design + + +1.10.2 Implementation Constraints + +• Evaluation and testing of software options is expected to occur prior to selection and + implementation of a production urban platform. +• Budget costs are unknown until evaluation of software options is completed. + + +1.11 Assumptions, Alignment with other Action Clusters and Policies +1.11.1 Assumptions + +The assumptions in Table 3 have been identified by the Demand Side Engagement Stream as +relevant to this Requirements Specification. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 11 + + +Table 3. Assumptions + + # ASSUMPTION + + 1 The providers of city data and services will be responsible to maintain their resources in the platform. + + 2 All city data must meet the minimum metadata requirements and use the standards adopted by the platform. + + 3 The platform shall consider open Source as an optional commercial model, with open standards as a principle + + 4 The system design and architecture should minimize fragmentation of city data in the urban platform. + + 5 To the extent possible, automation should be used for the extraction of descriptive and technical metadata. + + 6 The platform must be designed in a way it accommodates additional functionality at later stage at a fair and transparent cost. + + The platform must be a modular based architecture which relies on stable and well-defined open interfaces to ensure + 7 + interoperability between the platform, services and the applications provided by service providers. + + The platform will offer open and well-documented API’s and clear service descriptions and contracts that is offered for reuse by + 8 another party to foster open innovation in the city, which means that developers and interested individuals openly utilize the + resources provided. + + 9 Adopt open and published European and International standards where possible. + + The platform must be flexible enough to accommodate different local, National and International data protection, licensing and +10 + commercialization regulations. + +11 Platform providers will monitor emerging technologies in order to maintain and improve the architecture. + + +12 Platform providers will monitor emerging information standards, including metadata standards and data interface standards. + + +13 Platform providers will monitor new commercial models for city data exploitation + + + + +1.11.2 Alignment with Citizen’s Focus Action Cluster +This specification document is aligned with the principles defined in the Citizen Focus5 Action +Cluster of the European Innovation Partnership on Smart Cities and Communities. Citizen Focus’ is +about “working together with citizens to realize public interests at the intersection of ICT, mobility +and energy in an urban environment”. We recognize citizens as owners of and participants in +the creation and delivery of city data and digital services, and we specify requirements to deliver +new digital services that will address the societal needs of cities in a positive manner that relates to +political narratives. Societal needs and wants are considered the starting point for city data service +offering by the urban platform. The requirements in this document were elicited considering + +• Human behaviour and needs as important as technology; +• The services and data that solve social problems and drive innovation; +• The mechanisms that make data and services more accessible to users; +• The factors that influences user’s experience while interacting with services provided (e.g. + usability, feeling of security and trust); + + +5 + https://eu-smartcities.eu/node/1333 + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 12 + + +1.11.3 Policies to be developed + +The following policies have been identified by the Demand Side Engagement Stream as relevant to +this Requirements Specification. + +• Data formatting and Metadata Schemes: Urban platforms will require more expansive, robust + and useful data encoding and conversion that what is available in existing data catalogues. + Data preservation policies should be developed to allow data to be stored in formats that can + be migrated, associated with metadata and ontologies to become both humans and machines + readable and understandable, ongoing monitoring for data obsolescence, and migrating data to + systems environments as needed to ensure their continued availability. Current Metadata + schemes (e.g. open data, sensory data ontologies) should be reviewed to see if it meets + current needs for city data management. It is possible that the needs of the urban platform + designed here may require new or additional schemas. + +• Data commercialisation: The commercial exploitation of city data and their funding models + are unexplored concepts that we are committed to address. There is an urgent need to define + license agreements and fair commercial and subscription models to allow interoperable open + and proprietary data to co-exist in the platform. + +• Data publication and services deployment: Policy development will be needed regarding + ingesting data and deploying data services into the platform, including which users/machines + will be authorized to submit data for publication, the minimum requirements for data submitted + by open and proprietary data publishers, and the removal of resources and services from the + platform. + + +2. Urban Platform Value Proposition, Use Cases and +Functional Requirements +2.1 From Value Proposition to Platform Specifications +This document uses goal-oriented modelling for eliciting, elaborating, structuring, specifying, +documenting, and modifying requirements. Goals represent the objectives which the urban +platform should achieve through cooperation of actors in the intended system and in the +environment. They capture, at different levels of abstraction, the various objectives the urban +platform under design should achieve. Through goals modelling we consider how the value +proposition and intended solutions connects across the stack, how the urban platform meets city +goals, why the system and its functionality are needed, and how the stakeholders’ interests may be +addressed. + +In our specification, we present the overall goal (the value proposition) that the urban platform +should aim to achieve in order to be considered as a viable final product, and a set of sub-goals +(intended solutions) it should maintain in the long run so that the overall goal can be unceasingly +achieved. By using this approach, the low-level technical requirements can be traced back to high- +level strategic objectives of the urban platform. The formal notations used in this document are: +Achieve [“Name of Overall Goal”] and Maintain [“Name of Sub-Goal”]. The requirements of the +Urban Platform are noted for each of the sub-goals, and are presented as a series of statements +regarding the capabilities needed in to achieve the overall goal of the Urban Platform. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 13 + + +2.2 Overall Goal: “City data is exploited to its full potential” +An urban platform is an organization of people and systems, which has accepted the responsibility +to preserve city data and make it available for all the stakeholders of smart cities. Ultimately, an +urban platform is a foundation for the full exploitation of city data. Hence, the major goal an Urban +Platform must achieve is “city data is exploited to its full potential”. To achieve this high level goal, +the urban platform must maintain in the long run the five sub-goals illustrated in Figure 3. + +Each one of the defined sub-goals co-enables the achievement of the specified high-level (overall) +goal of the urban platform. The sub-goals include the ingestion of city data, metadata generation, +data management, data storage, access, preservation, and administration, provision of engaging +services in the smart cities. These sub-goals are discussed in details in the following sections. + + + + Achieve [City data is exploited How we achieve the platform overall goal? + Why do we need the sub-goals? + to its full effect] + Goal + + + + + Maintain [User s experience is + Maintain [City data is provided + enhanced by the provision of + in a harmonised way] + S ub-Goal 1 S ub-Goal 5 value-added services] + co-ena bles + + + + + Maintain [Resources are managed in Maintain [City data is offered + a safe and intelligent manner] in an accessible manner] + S ub-Goal 2 S ub-Goal 4 + + + + + Maintain [City data is orchestrated + in a market place] + S ub-Goal 3 + + + + + Figure 3. Platform High-Level Goal and its respective sub-goals. + + + +2.2.1 Urban Platform Boundary + +The use case diagram illustrated in Figure 4 identifies the boundaries between the actors (either +automated or human) and the urban platform. We have arrived at the urban platform boundary by +inspecting each business use case and determining, in conjunction with the stakeholders needs, +which part of the business use case should be implemented and what part should be done by an +outsourced product (e.g. Billing System) using the framework 4. + +This task is technology agnostic and takes into account the abilities of the users/actors, the +constraints, the goals of the urban platform. Table 4 maps out the use cases with their respective +sub-goals and actors. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 14 + + + + + Manage Services + + S ub-Goal 5 + + + Data Service + Publish City Data + Provider + S ub-Goal 1 + + + <> Deploy Data Services + rules + S ub-Goal 5 + + Authenticate in the + Manage Resources <> + Platform Platform + S ub-Goal 1 Provider Database + <> Manage Infrastructure System + S ub-Goal 2 + + Utlise Data Services + City Data <> + Publisher S ub-Goal 5 + + Store City Data Management + <> + S ub-Goal 2 + Systems Services + System + + <> + City Data + Consumer + QoS Monitoring + Transmit Data + System + Register in the Platform S ub-Goal 2 + + + + + <> + rules + + <> + rules + Platform + Provider + rules + + Commercialise Data + Consume City Data + Services City Data + S ub-Goal 3 S ub-Goal 4 + Consumer + <> + + + Commercialise + City Data Discover City Data + S ub-Goal 3 + S ub-Goal 4 + + Billing + Management + System + + + + Authenticate in the Register in the + Platform Platform + + + + + Authenticate Authenticate + Register Consumers Register Publishers + Consumers Publishers + + + + + Figure 4. Simplistic overview of the use cases identified in the early stages of the platform design. + + + + +Table 4. Use Cases Mapping with Sub-Goals and Actors + Sub-Goals Use Cases ID Specialised Use Cases Actors +HIGH-LEVEL GOAL: City data is exploited to its + + + + + Publish City Data UC1 User publishes city data via data API’s + City Data Publisher + 1. City data is Register as a publisher + UC2 User manually uploads datasets + provided in a in the Platform + harmonised way + full effect + + + + + User manages resources + Manage Resources UC3 City Data Publisher + User tracks resources usage + + Store City Data UC4 + 2. City data is + managed in a safe Transmit Data UC5 - Management Systems + and intelligent + manner + Manage Infrastructure UC6 + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 15 + + + City Data Publisher + Set commercial city data + Platform Provider + Subscribe to proprietary data City Data Consumer + Commercialise City + UC7 + Data + Manage commercial data City Data Publisher + + 3. City data is Manage data subscription City Data Consumer + orchestrated in a + Data Services Provider + market place Set commercial data services + Platform Provider + Subscribe to commercial services Data Services Consumer + Commercialise Data + UC8 + Services + Manage commercial services Data Services Provider + + Manage services subscription Data Services Consumer + UC9 + Register in the Platform City Data Consumer + - + 4. City data is Discover City Data City Data Consumer + UC10 + offered in an + User consumes city data via data + accessible manner + API’s + Consume City Data UC11 City Data Consumer + User downloads datasets + 5. User’s Data Services Provider + Deploy Data Services UC12 + experience is Platform Provider + enhanced by the + Manage Services UC13 - Data Services Provider + provision of value- + added services + Utilise Data Services UC14 City Data Consumer + + + + + 2.3 SUB-GOAL 1: City data is collected in an intelligent manner + Description: The urban platform enables the owners of city data to easily publish both historic and + data streams in the platform, as well as their associated metadata. + Rationale: This sub-goal is maintained by the services and functions to accept the publication of + city data from data providers (of both open and proprietary data) and prepare the contents for + storage and management within the urban platform. Functions include receiving data, performing + quality assurance on data, verifying data formatting and document standards, associating meta- + data information, and coordinating updates to databases and resources management. + + Achieve [City data is exploited + to its full effect] + Goal + co-ena bles + + + + + How we achieve the platform overall goal? + + + + Maintain [City data is provided in + a harmonised manner] + S ub-Goal 1 + + + + +What actions can maintain the sub-goal? + + + + Achieve [Register Publisher] Achieve [Publish City Data] Achieve [Manage Resources] + UC1 UC2 UC3 + + + + + Figure 5. Sub-Goal 1 “City data is collected in an intelligent manner” refinement. + + + Drivers: Ensure data is published in an easy and uniform way. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 16 + + +Actions: For Sub-Goal 1 to be maintained in the long-run it requires the efficient realisation of use +cases: “Publish City Data” and “Manage Resources”, as shown in Figure 5. + + +2.3.1 Use Case: Register Publisher +ID: UC1 +Refines: SUB-GOAL 1: City data is collected in an intelligent manner +Pre-condition: Data Publisher is not logged in the system +Actors: Data Publishers +Rationale: Data Publishers can register in the platform and request approval to submit city data. +They provide valid registration details (to be defined) and wait for registration confirmation. +Platform Providers may authorise or not data publishers to offer both open and proprietary city data +in the platform. Data submission agreement is a formal agreement between the Data Provider and +the Urban Platform defining the terms of the content, standards, metadata creation, and license +agreement. The Urban Platform will proactively work with Data Providers to agree on the content, +quality and format of city data. Agreements between Platform and Data Providers may be +renegotiated on a periodic or ad-hoc basis. +Refines into requirements: FREQ.1 to FREQ.5. + + Use Case Basic Stimulus and Responses + 1. The platform prompts the user for a username and password or + register new account. + 2. The user selects registration option. + 3. The platform prompts user for publisher registration information (e.g. + username, password, organisation) + 4. The user enters in their information. + UC1. Register 5. Platform verifies information and creates account. + Publisher o If non-valid information, platform shows error message and + returns to step 1. + 6. Platform provider is requested to approve the account + o Platform acknowledges registration has been successful + o If non approved, platform shows error message and returns to + step 1. + 7. End of registration + + +2.3.2 Use Case: Publish City Data +ID: UC2 +Refines: SUB-GOAL 1 - City data is collected in an intelligent manner +Pre-condition: User is authenticated in the platform +Actors: City data publisher +Rationale: The Publish City Data function provides the appropriate mechanisms to receive city data from +authorized data providers. Data may be manually uploaded or submitted via APIs. In general, data providers +with whom the Urban Platform negotiates submission agreements are the providers of proprietary city data +(those producing published material, i.e. publishers) and open data, and they can be either humans or +machines. The providers of the Urban Platform will provide data providers with specifications on the content, +quality and format of data, and publication terms and conditions. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 17 + + +The Publish City Data function may represent a legal transfer of custody for the data in the urban platform, +and may require that special access controls be placed on the contents. This function provides a +confirmation of receipt of data publication to the Producer, which may include a request to resubmit data in +the case of errors resulting from the submission. +Once data has arrived, it must undergo several reviews, including virus checking, format compliance, +metadata minimum requirement agreement, quality and anticipated content and data formatting. The +platform must include the ability to record all actions and decisions made concerning the publication of city +data. The reasons for publication failure (e.g. missing metadata information, non-valid dataset) will be +provided back to the city data publisher. In some cases, the provider can then resubmit corrected data and +metadata information, while in other instances data publication refusal criteria should prevent the publisher +from submitting the same dataset at a later time period (e.g. in cases of suspicious datasets – copyrights +violation, viruses). When data is successfully submitted (either via APIs or manual upload), it will be +processed/prepared for storage into the platform’s database. +Specialised Use Cases: The Use Case Publish City Data data is distinguished into two +specialised Use Cases: “User publishes city data via data API’s (UC2.1)” and “User manually +uploads datasets (UC2.2)”. +Subordinated Use Cases: “Store City Data (UC4)” +Refines into requirements: FREQ 6 to FREQ.26. + + + Specialised Use Cases Basic Stimulus and Responses + 1. Platform provides user with an interface for static data + publication + 2. User selects datasets to be uploaded + 3. User provides metadata associated with the data (license, + provenance, ownership, semantics) in accordance with defined + standards + UC2.1. User manually 4. User requests data publication + uploads datasets 5. Platform quickly process user’s request for data publication + 6. Platform validates data submitted + o If valid data, platform acknowledges data publication has + been successful + o If non-valid data, platform shows error message and returns + to step 1. + 7. End of data publication + 1. Platform provides user with an interface for real-time data + publication + 2. User input data API information + 3. User provides metadata associated with the data (license, + provenance, ownership, semantics) in accordance with defined + standards + UC2.2. User publishes 4. User confirm information and request data publication + city data via data API’s 5. Platform quickly process user’s request for data publication + 6. Platform validates data submitted + o If valid data, platform acknowledges data publication has + been successful + o If non-valid data, platform shows error message and returns + to step 1 + End of data publication + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 18 + + +2.3.3 Use Case: Manage Resources +ID: UC3 +Refines: GOAL 1: City data is collected in an intelligent manner +Pre-condition: User successfully authenticates in the platform +Actors: City data publisher +Rationale: Manage resources provides the services and functions for updating, maintaining and +accessing both data and metadata, as well as tracking the usage of resources by users. Ideally +the owners of the resources should be the only authorised user to manage resources, and other +authorised users can track the usage of the resources in the platform. The platform must provide a +database update response indicating the status of the update, avoid update errors to be +propagated in the platform, and should keep an audit trail of all actions to enable rollback. Data +usage tracking includes performing queries on the data management data to generate result sets, +and producing reports from these result sets. +Specialised Use Cases: The Use Case Manage Resources data is distinguished into two +specialised Use Cases: “User manages resources (UC3.1)” and “User tracks resources usage +(UC3.2)”. +Subordinated Use Cases: “Transmit Data (UC5)” +Refines into requirements: FREQ.27 to FREQ.25. + + + Specialised Use Cases Basic Interactions and Responses + 1. Platform provides user with an interface for resources + management (e.g. data and metadata, data usage) + 2. User chooses to edit or delete data + 3. If edit, user revise metadata associated with the data (license, + provenance, ownership, access-control, semantics); + UC3.1. User manages 4. If delete, user selects dataset(s) to be removed + resources 5. User confirms action + 6. Platform quickly process user’s request + 7. Platform confirms execution of request + o If valid request, platform acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 8. End of resources management + 1. Platform provides user with an interface for resources + management (e.g. data and metadata, data usage) + 2. User chooses to visualise usage information of a dataset + UC3.2. User tracks 3. Platform quickly process user’s request for data usage + resources usage information + 4. Platform provides user with statistical information about data + usage and data users anonymised information + 5. End of data usage tracking. + + +2.3.4 Functional Requirements + + Req. ID UC. ID Description Priority Domain + Societal Needs, + FREQ.1 UC1 Allow data publishers to register to submit data for publication Must + Platform + Tracks data publication agreements between Data and Platform Business Needs, + FREQ.2 UC1 Must + Providers Platform + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 19 + + + Store terms of agreements, and use them to monitor/review/process + FREQ.3 UC1 Must City Data, Platform + data submissions. + Able to add and edit terms of agreement, based on access of level of Business Needs, + FREQ.4 UC1 Must + user. Platform + + FREQ.5 UC1 Data publications are managed and monitored Must City Data, Platform + + Allow authenticated users from across different organisations to City Data, Platform, + FREQ.6 UC2 Must + publish city data Business Needs + Provide authorization mechanisms for users and sensors to publish + FREQ.7 UC2 Must City Data, Platform + city data + City Data, Platform, + FREQ.9 UC2 Provide mechanisms for static data publication Must + Business Needs + City Data, Platform, + FREQ.10 UC2 Provide mechanisms for real-time data publication Must + Business Needs + + FREQ.11 UC2 Enable the publication of metadata Must City Data, Platform + + + FREQ.12 UC2 Maintain temporal information about the data Must City Data, Platform + + + FREQ.13 UC2 Support sensory data collection Must City Data, Platform + + + FREQ.14 UC2 Accepts content in numerous file types/formats Must City Data, Platform + + Prompts a request for resubmission to the data provider if an error of + FREQ.15 UC2 Must City Data, Platform + data transmission or receipt occurs + + FREQ.16 UC2 Enable the semantic description of connected devices Must City Data, Platform + + + FREQ.17 UC2 Gather data from authenticated and authorized devices Must City Data, Platform + + + FREQ.18 UC2 Validates automatically the successful transfer of the data Must City Data, Platform + + + FREQ.19 UC2 Performs virus checking on data Must City Data, Platform + + Verifies the validity of the submission based on submitter, expected + FREQ.20 UC2 Must City Data, Platform + format, data quality, and completeness + Platform should have built-in checks on the incoming metadata. Data + FREQ.21 UC2 not containing the minimally defined set of attributes should be Must City Data, Platform + returned to the publisher for metadata enhancement. + System should have a user-friendly method of mapping non-standard + FREQ.22 UC2 Should City Data, Platform + metadata elements into approved standard elements. + Once ingested, metadata should be stored in a single common format. + FREQ.23 UC2 This format should be one that ensures against data loss, and allows a Must City Data, Platform + variety of access/distribution options + Data in the repository shall have sufficient technical metadata to + FREQ.24 UC2 assure functionality (e.g. viewing and display) to ensure accessibility Must City Data, Platform + and reusability. + Allows publishers to display and perform manual/visual quality control Business Needs, City + FREQ.25 UC2 Could + assurance via a user-friendly GUI Data, Platform + Business Needs, City + FREQ.26 UC2 Any errors shall prompt a request for resubmission of data Should + Data, Platform + + FREQ.27 UC3 Enable data providers to manage their resources Must Business Needs + + A minimal set of identifying information/metadata concerning data Business Needs, + FREQ.28 UC3 Must + publication submission must be recorded Platform + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 20 + + + Stores and tracks versions of data. Links /connections between + FREQ.29 UC3 Must City Data, Platform + versions are created and maintained + Give service and data providers access to anonymized data of the Business Needs + FREQ.30 UC3 Should + subscribers of their data or services + City Data, Platform, + FREQ.31 UC3 Enable data providers to maintain and repair data and metadata Should + Business Needs + Tracks data publication agreements between Data and Platform Business Needs, + FREQ.32 UC3 Must + Providers Platform + Store terms of agreements, and use them to monitor/review/process + FREQ.33 UC3 Must City Data, Platform + data submissions. + Able to add and edit terms of agreement, based on access of level of Business Needs, + FREQ.34 UC3 Must + user. Platform + + FREQ.35 UC3 Submission volumes and schedules are managed and monitored Must City Data, Platform + + + + +2.4 SUB-GOAL 2: City data is managed in a safe and intelligent manner +Rationale: The urban platform enables users to publish, consume and commercialise data, as well +as deploy and manage services all in a secure and privacy protected manner. + + Achieve [City data is exploited + to its full effect] + Goal + co-ena bles + + + + + Maintain [Resources are managed in a + safe and intelligent manner] + S ub-Goal 1 + + + + + Achieve [Store Data] Achieve [Transmit Data] Achieve [Manage Infrastructure] + UC4 UC5 UC6 + + + Figure 6. Sub-Goal 2 “City data is managed in a safe and intelligent manner” refinement. + + + +Drivers: Ensure data is secured and the identity of users are preserved +Actions: For Sub-Goal 2 to be maintained in the long-run it requires the efficient realisation of use +cases: “Store City Data” and “Retrieve and Transmit City Data”, as shown in Figure 6. + + +2.4.1 Use Case: Store City Data +ID: UC4 +Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner +Pre-condition: Data is successfully published in the platform +Actors: Urban Platform +Rationale: When data is successfully submitted (either via APIs or manual upload), it is +processed/prepared for storage into the platform’s database. This procedure will include the + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 21 + + +generation of unique identifiers to the database, enrichment with ontologies (when applicable), +encrypted (when applicable), signed with digital certificates (when applicable) to ensure that the +data conforms to the platform data formatting, standards, security and regulation. Data may be +converted to accepted formats, as needed (e.g. graph model). A primary goal of the conversion of +content for the platform is the preservation of the content. Priority will be given to preserving the +data accordingly to the policies defined in section (2.6.2). Access-control levels and license models +are associated to data which is subject to restrictions relating to access and conditions of use. +Refines into requirements: FREQ 28 – FREQ 39. + + Use Case Basic Interactions and Responses + 1. Platform mechanisms converts submitted data into a standard format + 2. Security enforcement (e.g. anonymisation, cryptography) is placed on + sensitive information. + UC4. Store City 3. Platform associates with datasets the access-control definitions set by + Data owner of resources. + 4. Data is enriched with semantics and is associated with other datasets + 5. Platform stores data in a scalable and secure database. + 6. End of data storage. + + +2.4.2 Use Case: Transmit Data +ID: UC5 +Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner +Pre-condition: Data is successfully published in the platform +Actors: Urban Platform +Rationale: The platform accepts data retrieval request, validates user’s rights to access the data, +retrieves city data from data storage, and moves a copy of the data to the relevant platform +component for further processing. If special processing is required, the retrieval function accesses +data in staging storage and applies the requested processes. The types of operations, which may +be carried out, include sub-sampling in temporal or spatial dimensions, conversions between +different data types or output formats, and other specialized processing (e.g., data visualisation). +Once it is finalised data will be sent to the appropriate delivery channels (e.g. API’s, GUI). It also +encompasses function to verify corruption during any internal data transfer. This function requires +that all hardware and software within the platform provide notification of potential errors and that +these errors are routed to standard error logs that are checked by the Platform Provider. +Refines into requirements: FREQ 40 to 54. +2.4.3 Use Case: Manage Infrastructure +ID: UC6 +Refines: SUB-GOAL 2: City data is managed in a safe and intelligent manner +Pre-condition: The platform is available +Actors: Urban Platform +Rationale: Manage infrastructure provides the services and functions for the overall operation of +the urban platform. Administration functions include monitoring quality of service agreements, +auditing data publication to ensure that they meet archive standards, and maintaining configuration +management of system hardware and software. In overall, it provides system engineering +functions to monitor and improve platform operations, and to inventory, report on, and +migrate/update the contents of the platforms’ databases. +Refines into requirements: FREQ 55 to 62. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 22 + + + + Use Case Basic Interactions and Responses + 1. Platform keeps monitoring services at run-time to ensure operation + and integrity of city data + o If system failure, the platform activates mechanisms for recovery + UC6. Manage based on pre-defined rules + Infrastructure o Platform logs issue and issue alert messages to platform providers + 2. Platform logs operation capabilities (e.g. performance, mean of time + failure, issues, etc.) + + +2.4.4 Functional Requirements + + Req. ID UC. ID Description Priority Domain + A minimal set of identifying information/metadata concerning data Business Needs, + FREQ.28 UC4 Must + publication submission must be recorded. Platform + Stores and tracks versions of data. Links /connections between + FREQ.29 UC4 Must City Data, Platform + versions are created and maintained. + + FREQ.30 UC4 Converts data to accepted file formats Must City Data, Platform + + Keep sensitive information secured and accessible only to authorized + FREQ.31 UC4 Must City Data, Platform + users + + FREQ.32 UC4 Keep user’s personal information protected Should City Data, Platform + + + FREQ.33 UC4 Keep city data and meta-data secured Must Platform Needs + + + FREQ.34 UC4 Enable privacy preserving mechanisms associated to data Must Platform Needs + + + FREQ.35 UC4 Model data in accordance with defined standards Must City Data, Platform + + + FREQ.36 UC4 Support the use of ontologies and semantic modelling of city data Could City Data + + + FREQ.37 UC4 Support database-level provenance annotation Should City Data + + + FREQ.38 UC4 Support data-level provenance annotation Should City Data + + + FREQ.39 UC4 Enable data to be encrypted Should Platform Needs + + System must have the ability to search and display metadata, preferably + City Data, Platform + FREQ.40 UC5 in a user-conformable, human readable display as well as in its native Must + Needs + format for machine harvesting and manipulation. + Controls access to data in the repository based on multiple permission + City Data, Platform + FREQ.41 UC5 levels. These permission levels determine the create/edit/read/delete Should + Needs + privileges granted users. + Access rights and conditions of use will be held for each data and its City Data, Platform + FREQ.42 UC5 Should + related metadata. Needs + Access rights and conditions can be inherited from a parent data to any City Data, Platform + FREQ.43 UC5 Could + data designated as a child data (derived information). Needs + Access rights and conditions of use will be machine readable and City Data, Platform + FREQ.44 UC5 Should + actionable. Needs + Access mechanisms must be sufficiently granular to allow the Business Needs, City + FREQ.45 UC5 identification of individual users, in order to maintain audit logs of actions Should + Data, Platform + performed by users. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 23 + + + Maintains the integrity of the database which contains both metadata + FREQ.46 UC5 and system information. Must Platform Needs + + Provides internal validation such as referential integrity of the contents of City Data, Platform + FREQ.47 UC5 the database. Must + Needs + Creates and maintains schema definitions required to support data + FREQ.48 UC5 management functions. Must Platform Needs + + Monitors and ensures that data and metadata are not corrupted during + FREQ.49 UC5 transfers. Must Platform Needs + + Provides statistically acceptable assurance that no components of the + FREQ.50 UC5 data are corrupted during any internal data transfer. Must Platform Needs + + Performs routine and special data integrity checking for each dataset + FREQ.51 UC5 and generates error reports. Must Platform Needs + + Provides disaster recovery capabilities including data backup, off-site + FREQ.52 UC5 data storage, data recovery, etc. Must Platform Needs + + Refresh/replace data without service interruption, and update + FREQ.53 UC5 corresponding metadata as appropriate. Must Platform Needs + + Ensure that any associated unique identifiers of the updated data are not + FREQ.54 UC5 altered. Must Platform Needs + + Audits submissions to ensure that they meet archive/repository + FREQ.55 UC6 standards. Must Platform Needs + + Maintains configuration management of the system hardware and + FREQ.56 UC6 software. Must Platform Needs + + Has capability to inventory, report on and migrate the contents of the + FREQ.57 UC6 repository. Must Platform Needs + + + FREQ.58 UC6 Ensures data integrity for version upgrades and format migration. Must Platform Needs + + + FREQ.59 UC6 Monitors functionality of the entire repository. Must Platform Needs + + + FREQ.60 UC6 Maintains integrity of system configuration. Must Platform Needs + + Audits system operations, performance and usage. + FREQ.61 UC6 Must Platform Needs + + Provides platform performance information and database holdings + FREQ.62 UC6 inventory reports Must Platform Needs + + + + +2.5 SUB-GOAL 3: City data is orchestrated in a marketplace +Rationale: The urban platform enables users to consume and publish data in a secure and privacy +protected manner. +Drivers: Ensure data is secured and the identity of users are preserved +Actions: For Sub-Goal 3 to be maintained in the long-run it requires the efficient realisation of use +cases: “Commercialise City Data” and “Commercialise Data Services”, as shown in Figure 7. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 24 + + + + Achieve [City data is exploited + to its full effect] + Goal + + + + + co-ena bles + Maintain [City data is orchestrated in + a marketplace] + S ub-Goal 3 + + + + + Achieve [Commercialise Data + Achieve [Commercialise City Data] + Services] + UC7 UC8 + + + + + Figure 7. Sub-Goal 3 “City data is orchestrated in a marketplace” refinement. + + + +2.5.1 Use Case: Commercialise City Data +ID: UC7 +Refines: SUB-GOAL 3: City data is orchestrated in a marketplace +Pre-condition: Data is successfully published in the platform, both publisher and consumers of +city data are authenticated in the platform, and there are billing capabilities available. +Actors: Data Publishers, Data Consumers +Rationale: The providers of city data can commercialise city data based on the policies and +financial models defined in the platform. After publishing their data, publishers can define which +data can be available as open data and which data should be available with the payment of a +subscription fee. Once publishers define which data is to be commercially exploited, the platform +will associate the data with their respective financial models and let it ready for subscription. City +data consumer chooses which data to purchase and is redirected to a billing interface where the +subscription payment is taken. The platform must provide an update response indicating the status +of the payment. If successful, data is ready available to be consumed by humans and machines, +otherwise the user can re-try the payment or cancel transaction. +Commercialise city data also involves the function of managing commercial data. It provides +services and functions for updating, maintaining and accessing both data and its respective +commercial transactions. Furthermore, it enables data providers to track the usage of commercial +data by users. Ideally the owners of the data should be the only authorised user to manage +resources, and other authorised users can track the usage of the data in the platform. Data usage +tracking includes performing queries on the data management data to generate result sets, and +producing reports from these result sets. Data consumers are provided with functions which enable +them to manage their subscriptions and financial transactions on the platform. These functions +include updating, maintaining and accessing financial transactions. For all these functions and +services, the platform must provide a database update response indicating the status of the +update, avoid update errors to be propagated in the platform, and should keep an audit trail of all +actions to enable rollback. +Specialised Use Cases: The Use Case Commercialise City Data is distinguished into four +specialised Use Cases: “Commercialise Data (UC7.1)”, “Consume Commercial City Data (UC7.2)”, +“Manage Commercial Data (UC7.3)” and “Manage Data Subscription (UC7.4)”. +Refines into requirements: FREQ 63 to 68. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 25 + + + + + Use Case Basic Interactions and Responses + 1. User selects the data to be commercialised + 2. User selects the commercial model for data consumption + UC7.1. 3. Platform validates selection + Commercialise 4. Platform associates data to subscription model + Data 5. Platform releases data for commercial exploitation in the marketplace + 6. End of data commercialisation set up. + 1. User selects the data to be subscribed to + 2. User request subscription to data + UC7.2. 3. Platform validates selection + Consume 4. Platform redirects user to billing system + Proprietary 5. Billing system deals with user request + Data o If successful, user is redirected to a GUI where data is ready to use + o If unsuccessful, user can try payment again or cancel request + 2. End of data subscription. + 1. Platform provides user with an interface for commercial data + management + 2. User chooses to edit or delete commercial data + o If edit, user revise commercial models, licenses, access-control, + semantics; + o If delete, user selects dataset(s) to be removed (following policies + UC7.3. Manage defined in the platform for data removal) + commercial 3. User confirms action + Data 4. Platform promptly process user’s request + 5. Platform confirms execution of request + o If valid request, platform acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 6. End of resources management + 1. Platform provides user with an interface for data subscription + management + 2. User chooses to edit or cancel data subscription + o If edit, user revise payment and subscription timeframe; + o If cancel, user selects dataset(s) to have subscription cancelled + UC7.4. Manage (following policies defined in the platform for data subscription) + data 3. User confirms action + subscription 4. Platform promptly process user’s request + 5. Platform confirms execution of request + o If valid request, platform updates acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 6. End of data subscription management + + + +2.5.2 Use Case: Commercialise Data Services +ID: UC8 +Refines: SUB-GOAL 3: City data is orchestrated in a marketplace +Pre-condition: Data is successfully published in the platform, both publisher and consumers of +city data are authenticated in the platform, and there are billing capabilities available. +Actors: Data Service Providers, Data Publishers and Consumers + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 26 + + +Rationale: After deploying data services in the platform, the providers of data services can choose +to commercialise services based on the policies and financial models defined in the platform. Once +data service providers define which service(s) is (are) to be commercially exploited, the platform +will associate the services with their respective financial models and let available in the platform +applications module ready for use. User (either publisher or consumer) chooses which data +services to use, and in case a charged service is selected the platform redirects the user to a +billing interface where the payment is taken. The platform must provide an update response +indicating the status of the payment. If successful, service is ready available to be used, otherwise +the user can re-try the payment or cancel transaction. Note that data service owners should be +able to waive the payment of tariff to certain users’ categories. +Data services providers are offered with functions to manage their commercial services. The +platform provides functions for updating, maintaining and accessing both service and its respective +commercial transactions. Furthermore, it enables data services providers to track the usage of +services by users. Services usage tracking includes performing queries on the platform to +generate result sets, and producing reports from these result sets. The consumers of data services +are provided with functions which enable them to manage their subscriptions and financial +transactions. These functions include updating, maintaining and accessing financial transactions. +For all these functions and services, the platform must provide a database update response +indicating the status of the update, avoid update errors to be propagated in the platform, and +should keep an audit trail of all actions to enable rollback. +Specialised Use Cases: The Use Case Commercialise Data Services is distinguished into four +specialised Use Cases: “Commercialise Data Services (UC8.1)”, “Consume Data Services +(UC8.2)”, “Manage Commercial Services (UC8.3)” and “Manage Services Subscription (UC8.4)”. +Refines into requirements: FREQ 68 to 73. + + Use Case Basic Interactions and Responses + 1. User selects the data services to be commercialised + 2. User selects the commercial model for data service usage based on the + category of users + UC8.1. 3. Platform validates selection + Commercialise 4. Platform associates data services to subscription model + Data Services 5. Platform enables data service to be commercially exploited in the + marketplace + 6. End of services commercialisation set up. + 1. User selects the data service to be subscribed to + 2. User request subscription to service + 3. Platform validates selection + UC8.2. 4. Platform validates user’s category its respective commercial models + Consume Data 4. If applicable, platform redirects user to billing system + Services 5. Billing system deals with user’s request + o If successful, user is redirected to a GUI where data is ready to use + o If unsuccessful, user can try payment again or cancel request + 6. End of data subscription. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 27 + + + 1. Platform provides user with an interface for commercial data + management + 2. User chooses to edit or delete commercial data + o If edit, user revise commercial models, licenses, access-control, + semantics; + o If delete, user selects dataset(s) to be removed (following policies + UC8.3. Manage defined in the platform for data removal) + commercial 3. User confirms action + services 4. Platform promptly process user’s request + 5. Platform confirms execution of request + o If valid request, platform acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 6. End of resources management + 1. Platform provides user with an interface for data subscription + management + 2. User chooses to edit or cancel data subscription + o If edit, user revise payment and subscription timeframe; + o If cancel, user selects dataset(s) to have subscription cancelled + UC8.4. Manage (following policies defined in the platform for data subscription) + services 3. User confirms action + subscription 4. Platform promptly process user’s request + 5. Platform confirms execution of request + o If valid request, platform updates acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 6. End of data subscription management + + +2.5.3 Functional Requirements + + Req. ID UC. ID Description Priority Domain + City Data, Platform, + FREQ.63 UC7 Support the commercialization of city data Should Business Needs + City Data, Platform, + FREQ.64 UC7 Enable users to subscribe to city data through the payment of a tariff Should Business Needs + City Data, Platform, + FREQ.65 UC7 Enable users to manage their data subscriptions Should Business + Provide platform providers mechanisms to define the terms and + FREQ.66 UC7 Must Platform Needs + conditions for platform data usage + Enable data providers to manage the subscription models of their City Data, Platform, + FREQ.67 UC7 Should + data Business + UC7/ City Data, Platform, + FREQ.68 Utilise secure and reliable billing and payment management systems Must + UC8 Business + City Data, Platform, + FREQ.69 UC8 Support the commercialization of data services Should Business Needs + Enable data providers to manage the commercial models of their City Data, Platform, + FREQ.70 UC8 Should + services Business Needs + Provide service providers mechanisms to define the terms and City Data, Platform, + FREQ.71 UC8 Should + conditions of platform services Business Needs + Allow users to pay a tariff for using certain advanced services (e.g. City Data, Platform, + FREQ.72 UC8 Should + Data manipulation, enrichment) Business Needs + City Data, Platform, + FREQ.73 UC8 Enable users to manage their data services subscriptions Should Business Needs + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 28 + + +2.6 SUB-GOAL 4: City data is offered in an accessible manner +Rationale: The urban platform provides city data in both human and machine (e.g. sensors, +actuators, systems) readable and understandable formats. +Drivers: Ensure data understandability and machine-to-machine data transaction. +Actions: For Sub-Goal 3 to be maintained in the long-run it requires the efficient realisation of use +cases: “Register Consumer”, “Discover City Data”, and “Consume City Data” as shown in Figure 8. + + Achieve [City data is exploited + to its full effect] + Goal + + + + + co-ena bles + Maintain [City data is offered in an + accessible manner] + S ub-Goal 1 + + + + + Achieve [Register Consumer] Achieve [Discover City Data] Achieve [Consume City Data] + UC9 UC10 UC11 + + + + + Figure 8. Sub-Goal 4 “City data is offered in an accessible manner” refinement. + + + +2.6.1 Use Case: Register Data Consumer +ID: UC9 +Refines: SUB-GOAL 4: City data is offered in an accessible manner +Pre-condition: User is not logged in the platform +Actors: Data Consumers +Rationale: Data Consumers can register in the platform and request approval to consume city data +via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to +confirm their registration. Users must accept the terms and conditions of platform usage and define +how their personal data can be used by the Platform Owner. Users can manage and alter their +registration information at any time they want to. +Refines into requirements: FREQ 64. + + Use Case Basic Stimulus and Responses + 1. The platform prompts the user for a username and password or register + new account. + 2. The user selects registration option. + 3. The platform prompts user for data consumer registration information + UC1. Register 4. The user enters in their information. + Consumer 5. Platform verifies information and creates account. + o If non-valid information, platform shows error message and returns to + step 1. + 6. Platform acknowledges registration has been successful + 7. End of registration + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 29 + + +2.6.2 Use Case: Discover City Data +ID: UC10 +Refines: SUB-GOAL 4: City data is offered in an accessible manner +Pre-condition: User has access to either platform GUI or API +Actors: Data Consumers +Rationale: Data Consumers can register in the platform and request approval to consume city data +via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to +confirm their registration. +Refines into requirements: FREQ 64. + + Use Case Basic Stimulus and Responses + 1. Users access specialised data query end-points (e.g. SPARQL) + 2. Users provides information for pre-defined parameters for search + 3. Users request data search + 4. Platform quickly process users request for data + UC10.1 o All queries are verified against access rights restrictions + Discover city o If restriction applies users are redirected to log in interfaces + data via data o Users provide credentials and log on the system + query end-points 5. Users are provided with query results on the end-point if access is + allowed + o If access is not not allowed platform issues an error message to the + user + 1. Users search city data via GUI + 2. Users inputs search parameters (e.g. key words, categories, formats, + publishers) + UC10.2 3. Users request data search + Discover city 4. Platform quickly process users request for data + data via data o All queries are verified against access rights restrictions + query end-points o If restriction applies users are redirected to log in interfaces + o Users provide credentials and log on the system + 5. Users are provided with query results on an interface + o If access is not allowed platform issues an error message to the + user + + + +2.6.3 Use Case: Consume City Data +ID: UC11 +Refines: SUB-GOAL 4: City data is offered in an accessible manner +Pre-condition: User has access to either platform GUI or API +Actors: Data Consumers +Rationale: Data Consumers can register in the platform and request approval to consume city data +via GUI or APIs. They provide valid registration details (to be defined) and wait for platform to +confirm their registration. +Refines into requirements: FREQ 64. +Subordinated Use Cases: “Consume proprietary city data” + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 30 + + + + Use Case Basic Stimulus and Responses + 1. Users request data to be formatted in a particular format supported by + UC11.1 the platform + Customise City 2. Platform quickly process users request for data formatting + Data 3. Mechanism for data conversion is called and process data + 4. Users are provided with data formatted as requested + 1. Users / Machines select data to be downloaded + 2. Users / Machines are redirected to authentication mechanism in case of + UC11.2 registration is needed for the particular dataset + Consume City o If authentication is successful, users are provided with requested + Data via GUI data streams + 3. Users are provided with requested data via APIs + 1. Users / Machines makes data request on the platforms API + 2. Users / Machines are redirected to authentication mechanism in case of + UC11.3 registration is needed for the particular dataset + Consume City o If authentication is successful, users are provided with requested + Data via APIs data streams + 3. Users are provided with requested data via APIs + + + +2.6.4 Functional Requirements + + Req. ID UC. ID Description Priority Domain + Allow users to register to use services and consume proprietary city Societal Needs, + FREQ.74 UC9 Should + data and open data (optional) Platform + Keep sensitive information secured and accessible only to Societal Needs, + FREQ.75 UC9 Should + authorized users Platform + Societal Needs, + FREQ.76 UC9 Provide authentication mechanisms for users Must Platform + Societal Needs, + FREQ.77 UC9 Keep user’s personal information protected Must Platform + Allow users to control which data they are willing to provide and how Societal Needs, + FREQ.78 UC9 Must + their data should be used Platform + Societal Needs, + FREQ.79 UC10 Allow users to format data in any supported data formats Must + Platform + The query request may require data to be sourced from different Societal Needs, + FREQ.80 UC10 Must + storage locations Platform + Allows query requests against all metadata used to manage the Societal Needs, + FREQ.81 UC10 repository. Should Platform + Provide users information about the legal aspects of the data Societal Needs, + FREQ.82 UC11 Must + (license, ownership) City Data + Societal Needs, + FREQ.83 UC11 Keeps an audit trail of all actions. Must City Data + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 31 + + +2.7 SUB-GOAL 5: User’s experience is enhanced by the provision of +value-added services +Rationale: The urban platform enables users to consume and publish data in a secure and privacy +protected manner +Drivers: Ensure data is secured and the identity of users are preserved +Actions: For Sub-Goal 5 to be maintained in the long-run it requires the efficient realisation of use +cases: “Deploy Data Services”, “Manage Services”, and “Utilise Data Services” as shown in Figure +8. + + Achieve [City data is exploited + to its full effect] + Goal + + + + + co-ena bles + Maintain [User s experience is enhanced by + the provision of value-added services] + S ub-Goal 1 + + + + + Achieve [Deploy Data Services] Achieve [Manage Services] Achieve [Utilise Data Services] + UC12 UC13 UC14 + + + + + Figure 9. Sub-Goal 5 “User’s experience is enhanced by the provision of value-added services” refinement + + + +2.7.1 Deploy Data Services +ID: UC12 +Refines: SUB-GOAL 5: User’s experience is enhanced by the provision of value-added services +Pre-condition: Data Services Providers are provided with credentials, are authorised to deploy +their services in the platform, and have access to technical specifications of the platform interfaces. +Actors: Data Services Providers +Rationale: Data Services can register in the platform and request approval to deploy services in +the platform. They provide valid registration details (to be defined) and wait for registration +confirmation. Platform Providers may authorise or not data services providers to offer both open +and proprietary services in the platform. Data services providers must formally agree with service +deployment agreement with the Urban Platform. This agreement defines terms of the content, +policies, regulations, license agreement. The Urban Platform will proactively work with Service +Providers to agree on the technical specifications of interfaces and platform openness level. +Agreements between Platform and Data Service Providers may be renegotiated on a periodic or +ad-hoc basis. +Refines into requirements: FREQ.1 to FREQ.5. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 32 + + + + Use Case Basic Stimulus and Responses + 1. User authenticates in the platform + 2. User is provided access to platform interfaces for service deployment + 3. User deploy services + 4. Platform checks compatibility and any technical issues arising from the + UC12. Deploy new service + Services 5. If approved the service is ready to be used and managed in the + platform + o If deployment is not approved, platform shows error message and + returns to step 1. + 6. End of deployment + + + +2.7.2 Use Case: Manage Services +ID: UC13 +Refines: GOAL 5: User’s experience is enhanced by the provision of value-added services +Pre-condition: User successfully authenticates in the platform +Actors: Data Services Providers +Rationale: Manage services provides the functions for updating, maintaining and accessing +services as well as tracking their usage by users. Ideally the owners of the services should be the +only authorised user to manage them, and other authorised users can track the usage of the +services in the platform. In case of updates the platform must log in the database update details +and send to service providers a response indicating the status of the update. The platform should +also ensure update errors are not propagated nor affect the health of other services provided in the +platform, and should keep an audit trail of all actions to enable rollback. Data services usage +tracking includes performing queries on the data management data to generate result sets, and +producing reports from these result sets. All user’s information provided to service providers must +follow regulations of data protection and the user’s defined rules for their data use. +Specialised Use Cases: The Use Case Manage Services is distinguished into two specialised +Use Cases: “User manages services (UC13.1)” and “User tracks service usage (UC13.2)”. +Subordinated Use Cases: “Transmit Data (UC5)” +Refines into requirements: FREQ.27 to FREQ.25. + + + Specialised Use Cases Basic Interactions and Responses + 1. Platform provides user with an interface for services management + 2. User chooses to edit or delete services + 3. If edit, user revise service information (access-control, + commercial models, parameters) and deployment; + If delete, user selects services to be removed / disabled + UC13.1. User manages 4. User confirms action + services 5. Platform quickly process user’s request + 6. Platform confirms execution of request + o If valid request, platform acknowledges request has been + processed successfully + o If non-valid request, platform returns to step 1. + 7. End of services management + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 33 + + + 1. Platform provides user with an interface for services management + 2. User chooses to visualise usage information of a service + 3. Platform quickly process user’s request for data usage + UC13.2. User tracks information + services usage 4. Platform provides user with statistical information about services + usage and data users anonymised information + 5. End of data services tracking. + + + +2.7.3 Use Case: Utilise Data Services +ID: UC2 +Refines: SUB-GOAL 1 - City data is collected in an intelligent manner +Pre-condition: User is authenticated in the platform +Actors: City data publisher +Rationale: Data Consumers can register in the platform and request approval to consume city data via GUI +or APIs. They provide valid registration details (to be defined) and wait for platform to confirm their +registration. +Subordinated Use Cases: “Commercialise Data Services (UC8)” +Refines into requirements: FREQ 6 to FREQ.26. + + + Specialised Use Cases Basic Interactions and Responses + 1. Users / Machines select data service to be utilised + 2. Users / Machines are redirected to authentication mechanism in + UC2.1. User utilises data case of registration is needed for the particular service + services o If authentication is successful, users are provided with + requested data streams + 3. Users are provided with requested service either via API or GUI + + +2.7.4 Functional Requirements + + Req. ID UC. ID Description Priority Domain + Provide stable and well-defined interfaces to ensure interoperability + Societal Needs, + FREQ.84 UC12 between the platform, services and the applications provided by Should Platform + services providers + Ensure the interfaces of the architecture are open to reduce entry Societal Needs, + FREQ.85 UC12 Should + barriers and integration issues Platform + Provide multi-purposed and network intelligent interfaces to Societal Needs, + FREQ.86 UC12 Must + providers and consumers of services Platform + Provide service providers mechanisms to define the terms and Societal Needs, + FREQ.87 UC12 Must + conditions of platform services deployment Platform + Business Needs, + FREQ.88 UC13 Provide statistical information of user’s feedback on service usage Must Platform + Allow users to use services to manipulate city data (e.g. Create Societal Needs, + FREQ.89 UC10 Must + mash ups, integrate) Platform + Allow users to provide feedback on usability, and quality of data Societal Needs, + FREQ.90 UC14 Must + and services provided by the platform Platform + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 34 + + + +3. Non-functional Requirements +3.1 Run-time Quality Requirements +3.1.1 Scalability Requirements + +Rationale: The ability of the system to execute its task within its expected performance profile and +to handle on-demand increased processing volumes of data and service requests +Drivers: Provide ready access to all data that underpins decision making processes in smart cities, +and accommodate users and data usage patterns +Refines into requirements: NFREQ.1, NFREQ.2, NFREQ.3, NFREQ.4 +Relevance: Urban platforms have ambitious performance requirements. Such platforms must cope +with user’s demand for data and services, capture real-time data that will be catalysed by a myriad +of sensors. The demand for urban platform is very likely to significantly increase over time. It is +very difficult to have clear performance characteristics due to the ubiquity, heterogeneity high +connectivity of devices and end users. + + SCALABILITY MEASURES + Actions Capture the performance requirements + Create service level agreements + Predict scalability using software simulation + Analyse the performance of the platform overtime + Conduct practical testing + Strategy Prioritize service and data requests + Distribute processing over time + Scale up or scale out as necessary + Reuse resources and results Partition and parallelize + Constantly monitor Quality of Service at runtime + + + +3.1.2 Availability and Reliability Requirements + +Rationale: The ability of the system to be fully or partly operational as and when required and to +effectively handle failures that could affect system availability +Drivers: Build a reliable foundation for “on demand” exploitation of data +Refines into requirements: NFREQ.5, NFREQ.6, NFREQ.7, NFREQ.8 +Relevance: Any system that has complex or extended availability requirements, complex recovery +processes, or a high profile (e.g., is visible to the public) + + AVAILABILITY AND RELIABILITY MEASURES + Actions Capture the availability requirements Produce the availability schedule + Estimate platform availability Estimate functional availability Assess against the + requirements Rework the architecture + Strategy Adopt fault-tolerant hardware + Use reliable infrastructure database + Log transactions + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 35 + + + Develop adaptive software to cope and recover from faults + Design and test for failure + Deploy load balancing + Identify suitable backup and disaster recovery solution + + +3.1.3 Trust Requirements + +Rationale: A quality related to the user’s belief in the reliability, integrity and ability of the functional +behaviour of the platform +Drivers: Gain understanding of what influences user’s experience while interacting with services +provided +Requirements: NFREQ.16, NFREQ.17 +Relevance: Relevant to the systems that share and collect information that may raise public +concern. In some cases, trust has to do with the reliability of data and their providers, whereas in +other cases trust can be associated with the security and privacy of the technology that was +deployed. Trust affects the reputation of the platform besides its dissemination and maturity on the +market. + +TRUST MEASURES + Capture trust requirements + Perform risk analysis so measures can be implemented +Actions Explore the vulnerability aspects of city data + Check whether extensibility requirements impact on trust + Define a trust model + Adopt trust model + Deploy monitoring capabilities and assess its impact on scalability + Manage the data in a way that ensures its compliance with data +Strategy protection regulations + Implement tampering and data misuse detection + Make use of Cryptography when necessary + Allow Federation of trust between platforms + + + +3.1.4 Security Requirements + +Rationale: Ability of the system to enforce the intended confidentiality, trust, integrity and service +and data access policies, and to detect and recover from failure in these security mechanisms. +Drivers: Manage the data and services in a way that ensures its integrity, and compliance with +data protection regulations +Relevance: Relevant to the systems that share and collect information that may raise public +concern. Urban platforms may become a valuable target for attackers which can potentially leave +huge swathes of information exposed. It could potentially undermine trust in the government and +damage its reputation. +Refines into requirements: NFREQ.9 to NFREQ.15 + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 36 + + + SECURITY MEASURES + Elicit the security requirements + Cross-check security requirements’ impact on services offered by different + providers + Verify security impact on service composition + Conduct risk analysis and impacts of security breach + Use scalable and efficient user authentication components + Use authentication and authorization components to secure interfacing with + Actions external services + Address aspects of data collection, storage and distribution + Address aspects of service and communication security among devices + Identify security requirements of the physical infrastructure + Balance and prioritize scalability (performance) and security requirements + Evaluate trade-offs between privacy considerations and preventing abuse. (While + anonymous access guarantees privacy of users, traceability of users due to abuse + of the service is not possible.) + Toughen user’s functional components + Authenticate subjects + Define and enforce access policies + Secure communication infrastructures + Secure interfaces with external systems and services + Strategy + Secure databases + Check data integrity for critical services + User digital certificates and encryption where necessary + Secure monetary transactions + Secure user’s personal information + + +3.1.5 Privacy Requirements + +Rationale: Ability of the system to ensure that the collection and transmission of personal data is +minimized and handled in accordance with user’s expectation and regulations. +Drivers: Protect the vulnerability aspects volunteered citizen’s data +Requirements: NFREQ.18, NFREQ.19, NFREQ.20, NFREQ.21, NFREQ.22 +Relevance: Ensuring users privacy is protected positively influences user’s experience, +acceptance and continuous use of the platform. Besides other factors, the reputation of the +platform depends on how well user’s information is secure and preserved. + + PRIVACY MEASURES + Capture trust requirements + Perform risk analysis so measures can be implemented + Actions Explore the vulnerability aspects of city data + Check whether extensibility requirements impact on trust + Define a trust model + Allow users to interact with the platform anonymously in given + circumstances + Manage users identification securely + Strategy Anonymize data whenever necessary to comply with data regulations + Cryptograph personal identifiers during data transmission when that is + necessary + Avoid unauthorized access to implicit information (e.g. location) + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 37 + + + Verify the impact of security, trust and scalability requirements trade-offs + on privacy + Allow the user to control how personal information is used + Empower user to control the data disclosure mechanism + + + +3.2 Non Run-time Quality Requirements +3.2.1 Evolvability Requirements + +Rationale: The ability of the platform to withstand and easily adapt when new requirements and +changes is introduced. +Drivers: Ensure the platform is able to accommodate additional functionality and emerging +technologies at later stage at a fair and transparent cost +Refines into requirements: NFREQ.23 +Relevance: Important for longer- lived and more widely used systems. Urban platforms are +expected to be highly evolvable in order to accommodate future emerging technologies and avoid +interoperability issues. + + EVOLVABILITY MEASURES + Actions Characterize and assess the evolution needs + Consider the evolution trade-offs + Balance and negotiate potential conflicting requirements emerging from + the evolution capability. + Strategy Adopt standards and open interfaces + Design loose-coupled components + Preserve the platform resilience + + + +3.2.2 Extensibility Requirements + +Rationale: The flexibility of the system to allow services and functionality to be extended and +augmented by service providers in order to increase value of services to both platform providers +and end-users. The extension of the platform services is determined by the Platform Openness +strategy. +Drivers: Stakeholders can extend the services provided by the urban platform, so that +partnerships can be built to deliver holistic and interoperable solutions. Identify integrated +approaches to design and service delivery which ensures that services fit together and that +synergies can be exploited. +Refines into requirements: NFREQ.24, NFREQ.25 +Relevance: Important for longer- lived and more widely used systems. Urban platforms are +expected to be highly evolvable in order to accommodate future emerging technologies and avoid +interoperability issues. + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 38 + + + EXTENSIBILITY + Actions Characterize and assess the extensibility needs + Trace the impacts and cost of extensions + Negotiate potential conflicting requirements emerging from adding extensions + in the platform + Strategy Develop a modular architecture with standard interfaces that reduces entry + barriers due to increased transparency and integration + Share technical information about interfaces will help service providers in + targeting opportunities around the platform + + + +3.3 List of Non-Functional Requirements + +ID # Description Concern Priority Domain + + Platform, +NFREQ.1 Support different service level agreements (SLA) Scalability Should + Infrastructure + Platform, +NFREQ.2 Process services and events on a set of distributed nodes Scalability Should + Infrastructure + Platform, +NFREQ.3 Continuously monitor Quality of Service at runtime Scalability Should + Infrastructure + Platform, +NFREQ.4 Balance its load at runtime Scalability Should + Infrastructure + Platform, +NFREQ.5 Provide high availability Availability Should + Infrastructure + Platform, +NFREQ.6 Guarantee infrastructure availability Availability Should + Infrastructure + Platform, +NFREQ.7 Ensure network availability Availability Should + Infrastructure + Platform, +NFREQ.8 Be able to perform self-healing Availability Should + Infrastructure + City Data, +NFREQ.9 Expose data and services to authorized users Security Must + Platform + City Data, +NFREQ.10 Ensure services are always accessible to entitled users Security Must + Platform + +NFREQ.11 Ensure Data Freshness Security Must Platform + +NFREQ.12 Support access control mechanisms Security Must Platform + + Platform, +NFREQ.13 Have security mechanisms to protect data transmission Security Should + Infrastructure + Platform, +NFREQ.14 Make it difficult to spy on communicated messages Security Should + Infrastructure + Platform, +NFREQ.15 Be able to perform to detect threats at runtime Security Should + Infrastructure + Provide trusted and secure communication and information Platform, +NFREQ.16 Trust Should + management Infrastructure + +NFREQ.17 The platform infrastructure and services shall be trustable Trust Should Infrastructure + + Societal Needs, +NFREQ.18 Allow users to use free services anonymously Privacy Should + Platform + Societal Needs, +NFREQ.19 Allow people to use free services anonymously Privacy Should + Platform + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 39 + + + Allow users to control which data they are willing to provide Societal Needs, +NFREQ.20 Privacy Should + and how their data should be used Platform + +NFREQ.21 Keep users access-control rights/ policies secured. Privacy Should Platform + + Provide privacy protection for users interacting with the Societal Needs, +NFREQ.22 Privacy Should + platform Platform + Platform, +NFREQ.23 Provide communication confidentiality Privacy Should + Infrastructure + Societal Needs, +NFREQ.24 Be extensible for future technologies. Evolvability Must City Data, Platform, + Infrastructure + Platform, +NFREQ.26 Provide standard interfaces for service providers Extensibility Should + Business Needs + +NFREQ.26 Be able to provide services in an interoperable manner Extensibility Should Platform + + + + +4. Other Requirements +4.1 Minimal Descriptive Metadata Required from Data Provider + +Minimal Descriptive Metadata Required from Data Provider + + Attribute Required Definition Notes Examples + +Database Name Y A name given to the resource + +Author Name of a person or body associated with + Y + the creation of the resource + +Maintainer Name of the entity responsible for making + Y + the resource available + +Date Created Y Date the dataset was created + + +Date Modified Y Date the dataset was modified + + +Last Revision Y Date the dataset was last revised + + +Update Frequency Y Frequency of data maintenance + + +Mode of Release Y Open/Proprietary + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 40 + + +4.2 Urban Platform Supported Data Formats + + +MIME type Description Extensions Level + +application/json JavaScript Object Notation json supported + +application/xml eXtensible markup language file xml supported + +text/csv Comma separated values file csv supported + + + + +5. Conclusion & Forward Plans +This document represents the first set of requirements specification for urban platform. Future +activities will collaboratively assess, resolve requirements conflicts, prioritize, and validate the +requirements of the urban platform. Ultimately, this document will become a complete final +requirements specification document to guide and speed up the development open platform for +cities. Table below shows the milestones, deliverables and engagement activities completed and +yet to be completed by the Demand Side Engagement Stream. + + + + Milestones, deliverables and engagement activity Forecast date Status + Stakeholders invitation Late Completed + September/2015 + Online Workshop 1: Project Scope and Outcomes 20/11/2015 Completed + Online meeting held with city members to communicate goals, + expected outcomes, time commitment and required actions. + Start of Collaborative Requirements Engineering Process 20/11/2015 Completed + Participants reviewed the first draft of urban platform requirements, + provided comments, and suggested new requirements and changes. + Online Workshop 2: Review and Refine Requirements 08/12/2015 Completed + Participants had a conference call to review the requirements + specification document and provide their comments. + Deliverable: Requirements Specification Document v2.1 shared 05/01/2016 Completed + across the Working Streams for consultation + Online Workshop 3: Review Requirements Specification 12/01/2016 Completed + Document and discussion of Letter of Intent + Participants had a conference call to review the requirements + specification document and provide their comments, and discussed + the Letter of Intent to be signed by Mayor/Equiv of EU cities to + commit to application of common U.P. approach. + Urban Platforms Workshop in Brussels 19/01/2016 Completed + Participants provided feedback on the requirements specification + document and recommended changes in the document + Requirements Specification for Urban Platforms (EIP_SCC Initiative) Page 41 + + + Deliverable: Updated Requirements Specification Document v2.2 29/01/2016 Completed + shared with Industry and Standardisation Working Streams + Deliverable: Online release of the Requirements Specification Early + Document and open calls for wider consultation on the Requirements February/2016 + + Online Workshop 4: Engagement with Industry Side Early + Capture industry feedback on the Requirements Specification February/2016 + Document to assess its suitability to drive the development of the first + draft reference architecture. + Online Workshop 5: Review Requirements Specification Mid + Document v2.2 and discuss requirement prioritization and February/2016 + balancing + Collectively determine which candidate requirements of urban + platforms should be prioritized as high importance to all the cities + (use of Value Oriented Prioritization Method). Requirements are also + prioritized to minimize risk during development of urban platforms so + that the most important requirements are made explicit and + considered in the reference architecture. + Capture of case studies from cities intended to support requirements Mid March/2016 + validation (City of Porto), and learning and confidence building + amongst cities. + \ No newline at end of file diff --git a/internal/agent/orchestrator.go b/internal/agent/orchestrator.go index 7b7d19b..074a31e 100644 --- a/internal/agent/orchestrator.go +++ b/internal/agent/orchestrator.go @@ -3,6 +3,7 @@ package agent import ( "context" "fmt" + "runtime" "sort" "strconv" "strings" @@ -32,6 +33,29 @@ type Orchestrator struct { enableCapabilityGap bool log *logger.Logger skillsMu sync.RWMutex + pendingFilesMu sync.Mutex + pendingFiles map[string][]pendingFileRef +} + +type pendingFileRef struct { + ID string + Name string + MimeType string +} + +type capabilityRoutingResult struct { + NeedSkills bool + SelectedToolNames []string + SelectedSkills []knowledge.Skill + Reason string + UsedFallback bool +} + +type filePromptContext struct { + Summary string + FatalReason string + FileIDs []string + Uploaded []pendingFileRef } // NewOrchestrator 创建一个新的编排器对象，初始化关键路径和超时控制等。 @@ -76,6 +100,7 @@ func NewOrchestrator( reactMaxStep: reactMaxStep, enableCapabilityGap: enableCapabilityGap, log: log, + pendingFiles: make(map[string][]pendingFileRef), } } @@ -85,12 +110,20 @@ func NewOrchestrator( // - 是否需要调用工具（action + action_input） // 循环持续进行，直到 LLM 返回 is_final_answer=true。 func (o *Orchestrator) HandleMessage(ctx context.Context, chatID, userID, text string) (string, error) { + return o.handleMessageInternal(ctx, chatID, userID, text, nil) +} + +func (o *Orchestrator) HandleMessageWithFiles(ctx context.Context, chatID, userID, text string, files []llm.InputFile) (string, error) { + return o.handleMessageInternal(ctx, chatID, userID, text, files) +} + +func (o *Orchestrator) handleMessageInternal(ctx context.Context, chatID, userID, text string, files []llm.InputFile) (string, error) { // 为链路追踪设置唯一的 TraceID traceID := logger.NewTraceID() ctx = logger.WithTraceID(ctx, traceID) traceLogPrefix := "trace_id=" + traceID if o.log != nil { - o.log.Infof("%s handle message chat_id=%s user_id=%s text_len=%d", traceLogPrefix, chatID, userID, len(text)) + o.log.Infof("%s handle message chat_id=%s user_id=%s text_len=%d files=%d", traceLogPrefix, chatID, userID, len(text), len(files)) o.log.Debugf("%s handle message text=%q", traceLogPrefix, text) } @@ -111,6 +144,38 @@ func (o *Orchestrator) HandleMessage(ctx context.Context, chatID, userID, text s return report, nil } + trimmedText := strings.TrimSpace(text) + isFileOnly := len(files) > 0 && trimmedText == "" + + if isFileOnly { + if err := o.store.SaveMessage(chatID, userID, "user", "[FILE_UPLOAD]"); err != nil { + if o.log != nil { + o.log.Errorf("%s save file-only user marker failed chat_id=%s err=%v", traceLogPrefix, chatID, err) + } + return "", err + } + uploadCtx := o.prepareFilePromptContext(ctx, files, nil) + if strings.TrimSpace(uploadCtx.FatalReason) != "" { + finalText := "文件上传失败，无法建立文档上下文。" + "\n" + uploadCtx.FatalReason + if err := o.store.SaveMessage(chatID, userID, "assistant", finalText); err != nil && o.log != nil { + o.log.Warnf("%s save upload failure message failed chat_id=%s err=%v", traceLogPrefix, chatID, err) + } + return finalText, nil + } + o.appendPendingFiles(chatID, userID, uploadCtx.toPendingRefs()) + finalText := o.buildFileUploadAck(uploadCtx) + if err := o.store.SaveMessage(chatID, userID, "assistant", finalText); err != nil { + if o.log != nil { + o.log.Errorf("%s save file upload ack failed chat_id=%s err=%v", traceLogPrefix, chatID, err) + } + return "", err + } + if o.log != nil { + o.log.Infof("%s file-only message handled chat_id=%s cached_files=%d", traceLogPrefix, chatID, len(uploadCtx.FileIDs)) + } + return finalText, nil + } + // 保存用户消息到 SQLite 中 if err := o.store.SaveMessage(chatID, userID, "user", text); err != nil { if o.log != nil { @@ -133,13 +198,30 @@ func (o *Orchestrator) HandleMessage(ctx context.Context, chatID, userID, text s } // 进入统一 ReAct 循环 - response, err := o.runUnifiedReAct(ctx, chatID, userID, compressed, text) + pendingRefs := o.getPendingFiles(chatID, userID) + fileCtx := o.prepareFilePromptContext(ctx, files, pendingRefs) + if strings.TrimSpace(fileCtx.FatalReason) != "" { + finalText := "文件上传失败，无法继续进行文档解析。" + "\n" + fileCtx.FatalReason + if err := o.store.SaveMessage(chatID, userID, "assistant", finalText); err != nil && o.log != nil { + o.log.Warnf("%s save assistant failure message failed chat_id=%s err=%v", traceLogPrefix, chatID, err) + } + if o.log != nil { + o.log.Warnf("%s stop before react due to file upload failure reason=%s", traceLogPrefix, fileCtx.FatalReason) + } + return finalText, nil + } + routeInput := composeRouteInput(text, fileCtx.Summary) + route := o.routeCapabilities(ctx, routeInput) + response, err := o.runUnifiedReAct(ctx, chatID, userID, compressed, text, fileCtx, routeInput, route) if err != nil { if o.log != nil { o.log.Errorf("%s message generation failed chat_id=%s err=%v", traceLogPrefix, chatID, err) } return "", err } + if len(pendingRefs) > 0 { + o.clearPendingFiles(chatID, userID) + } // 最终将机器人的回复也加入记忆缓存 if err := o.store.SaveMessage(chatID, userID, "assistant", response); err != nil { @@ -156,21 +238,29 @@ func (o *Orchestrator) HandleMessage(ctx context.Context, chatID, userID, text s } // buildUnifiedSystemPrompt 构建统一 ReAct 循环的 system prompt。 -// 包含人格设定、所有可用技能（含完整内容）、所有可用工具、以及 JSON 输出格式约束。 -func (o *Orchestrator) buildUnifiedSystemPrompt() string { +// 工具始终可用；技能仅按当前问题挑选相关项作为增强上下文。 +func (o *Orchestrator) buildUnifiedSystemPrompt(userInput string, route capabilityRoutingResult) string { skillMetaDoc := o.formatSkillSummariesForPrompt() - allSkillsDoc := o.formatAllSkillsContent() + relevantSkillsDoc := o.formatSelectedSkillsForPrompt(userInput, route.SelectedSkills) toolDoc := o.formatToolDoc() + runtimeDoc := formatRuntimeContextForPrompt() + routeDoc := formatRouteForPrompt(route) return strings.Join([]string{ "你是一个个人自动化助手，必须遵循如下人格设定并保持一致：", o.soul, "", + "===== 运行环境 =====", + runtimeDoc, + "", "===== 可用技能概览 =====", skillMetaDoc, "", - "===== 技能详细说明 =====", - allSkillsDoc, + "===== 能力路由结果 =====", + routeDoc, + "", + "===== 本轮相关技能（按用户问题筛选） =====", + relevantSkillsDoc, "", "===== 可用工具 =====", toolDoc, @@ -190,25 +280,32 @@ func (o *Orchestrator) buildUnifiedSystemPrompt() string { "决策规则：", "1) 如果你可以直接回答用户问题（不需要任何工具）：", " 设 is_final_answer=true，action=\"none\"，final_answer 填写完整回复。", - "2) 如果你需要调用工具获取信息后才能回答：", + "2) 优先判断是否可通过原子工具能力完成任务；若可完成，直接进行工具调用链路。", + "3) 当纯工具调用无法满足时，再结合已加载的技能详细说明进行决策。", + "4) 如果你需要调用工具获取信息后才能回答：", " 设 is_final_answer=false，action 填工具名，action_input 填工具所需输入，final_answer=null。", - "3) 不要在 JSON 之外输出任何内容。", - "4) 根据技能说明中的指引决定何时以及如何使用工具。", - "5) 每轮工具调用结果会以 Observation 的形式追加到推理记录中，供你下一轮决策参考。", + "5) 不要在 JSON 之外输出任何内容。", + "6) 根据技能说明中的指引决定何时以及如何使用工具。", + "7) 工具能力是全局可用的，不依赖技能命中；当技能不匹配时，仍可直接选择合适工具。", + "8) 若技能中存在与当前运行环境不匹配的章节（如 Windows 专章），应降低优先级，除非用户明确要求该环境。", + "9) 每轮工具调用结果会以 Observation 的形式追加到推理记录中，供你下一轮决策参考。", }, "\n") } // runUnifiedReAct 执行统一的 ReAct 循环。 // LLM 每次都看到完整的技能集+工具集，自行决定是否调用工具或直接回答。 // 循环持续到 is_final_answer=true 或达到安全上限。 -func (o *Orchestrator) runUnifiedReAct(ctx context.Context, chatID, userID, compressedContext, userInput string) (string, error) { +func (o *Orchestrator) runUnifiedReAct(ctx context.Context, chatID, userID, compressedContext, userInput string, fileCtx filePromptContext, routeInput string, route capabilityRoutingResult) (string, error) { traceID := logger.TraceIDFromContext(ctx) traceLogPrefix := "trace_id=" + traceID - systemPrompt := o.buildUnifiedSystemPrompt() + if strings.TrimSpace(routeInput) == "" { + routeInput = composeRouteInput(userInput, fileCtx.Summary) + } + systemPrompt := o.buildUnifiedSystemPrompt(routeInput, route) if o.log != nil { - o.log.Infof("%s unified react start", traceLogPrefix) + o.log.Infof("%s unified react start route_need_skills=%v route_tools=%v route_skills=%d fallback=%v", traceLogPrefix, route.NeedSkills, route.SelectedToolNames, len(route.SelectedSkills), route.UsedFallback) } // 安全上限：防止无限循环（当前暂不使用 reactMaxStep 配置约束，使用固定硬上限） @@ -229,13 +326,16 @@ func (o *Orchestrator) runUnifiedReAct(ctx context.Context, chatID, userID, comp "用户问题：", userInput, "", + "文件上下文：", + defaultIfEmpty(fileCtx.Summary, "(none)"), + "", "当前推理记录（按时间顺序）：", scratchpad, "", "请输出你的 JSON 决策。", }, "\n") - raw, err := o.llm.Generate(ctx, systemPrompt, prompt) + raw, err := o.generateWithOptionalFiles(ctx, systemPrompt, prompt, fileCtx.FileIDs) if err != nil { return "", err } @@ -335,15 +435,514 @@ func (o *Orchestrator) runUnifiedReAct(ctx context.Context, chatID, userID, comp return "我尝试了多轮推理与工具调用，但仍未得到稳定结论。请给我更具体的约束或允许我继续尝试。", nil } -// formatAllSkillsContent 返回所有技能的完整内容，用于注入到 system prompt 中。 -func (o *Orchestrator) formatAllSkillsContent() string { - skills := o.getSkillsSnapshot() +func composeRouteInput(userInput, fileSummary string) string { + userInput = strings.TrimSpace(userInput) + fileSummary = strings.TrimSpace(fileSummary) + if userInput == "" { + return fileSummary + } + if fileSummary == "" { + return userInput + } + return userInput + "\n\n" + fileSummary +} + +func (o *Orchestrator) prepareFilePromptContext(ctx context.Context, files []llm.InputFile, pending []pendingFileRef) filePromptContext { + ctxOut := filePromptContext{} + if len(pending) > 0 { + for _, p := range pending { + id := strings.TrimSpace(p.ID) + if id == "" { + continue + } + ctxOut.FileIDs = append(ctxOut.FileIDs, id) + } + } + if len(files) == 0 { + ctxOut.Summary = buildFileSummary(pending, nil) + return ctxOut + } + uploader, ok := o.llm.(llm.FileUploader) + if !ok { + return filePromptContext{FatalReason: "检测到文件输入，但当前 LLM 客户端不支持文件上传接口。"} + } + + uploaded := make([]pendingFileRef, 0, len(files)) + for i, f := range files { + if strings.TrimSpace(f.FileName) == "" || len(f.Content) == 0 { + return filePromptContext{FatalReason: fmt.Sprintf("file[%d] 缺少文件名或内容，无法上传。", i+1)} + } + fileID, err := uploader.UploadFile(ctx, f, "file-extract") + if err != nil { + return filePromptContext{FatalReason: fmt.Sprintf("file[%d] name=%s 上传失败: %v", i+1, f.FileName, err)} + } + ctxOut.FileIDs = append(ctxOut.FileIDs, fileID) + uploaded = append(uploaded, pendingFileRef{ + ID: fileID, + Name: strings.TrimSpace(f.FileName), + MimeType: defaultIfEmpty(strings.TrimSpace(f.MimeType), "application/octet-stream"), + }) + } + ctxOut.Uploaded = uploaded + ctxOut.Summary = buildFileSummary(pending, uploaded) + return ctxOut +} + +func buildFileSummary(pending, uploaded []pendingFileRef) string { + if len(pending) == 0 && len(uploaded) == 0 { + return "" + } + lines := make([]string, 0, len(pending)+len(uploaded)+2) + lines = append(lines, "以下文件 file_id 可用于本轮问答：") + idx := 1 + for _, p := range pending { + id := strings.TrimSpace(p.ID) + if id == "" { + continue + } + lines = append(lines, fmt.Sprintf("- cached_file[%d] name=%s mime=%s file_id=%s", idx, defaultIfEmpty(strings.TrimSpace(p.Name), "(unknown)"), defaultIfEmpty(strings.TrimSpace(p.MimeType), "application/octet-stream"), id)) + idx++ + } + for _, p := range uploaded { + id := strings.TrimSpace(p.ID) + if id == "" { + continue + } + lines = append(lines, fmt.Sprintf("- uploaded_file[%d] name=%s mime=%s file_id=%s", idx, defaultIfEmpty(strings.TrimSpace(p.Name), "(unknown)"), defaultIfEmpty(strings.TrimSpace(p.MimeType), "application/octet-stream"), id)) + idx++ + } + if len(lines) == 1 { + return "" + } + return strings.Join(lines, "\n") +} + +func (o *Orchestrator) generateWithOptionalFiles(ctx context.Context, systemPrompt, userPrompt string, fileIDs []string) (string, error) { + ids := nonEmptyIDs(fileIDs) + if len(ids) == 0 { + return o.llm.Generate(ctx, systemPrompt, userPrompt) + } + client, ok := o.llm.(llm.FileChatClient) + if !ok { + return o.llm.Generate(ctx, systemPrompt, userPrompt) + } + return client.GenerateWithFiles(ctx, systemPrompt, userPrompt, ids) +} + +func (o *Orchestrator) buildFileUploadAck(ctx filePromptContext) string { + if len(ctx.FileIDs) == 0 { + return "文件已接收，但未拿到有效 file_id。请重新上传一次。" + } + lines := []string{ + fmt.Sprintf("文件上传完成，已缓存 %d 个 file_id。", len(ctx.FileIDs)), + "请继续发送你的问题，我会结合这些文件内容和历史对话一起回答。", + } + if strings.TrimSpace(ctx.Summary) != "" { + lines = append(lines, "", ctx.Summary) + } + return strings.Join(lines, "\n") +} + +func nonEmptyIDs(ids []string) []string { + if len(ids) == 0 { + return nil + } + out := make([]string, 0, len(ids)) + seen := map[string]struct{}{} + for _, id := range ids { + id = strings.TrimSpace(id) + if id == "" { + continue + } + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + out = append(out, id) + } + return out +} + +func (c filePromptContext) toPendingRefs() []pendingFileRef { + if len(c.Uploaded) > 0 { + copied := make([]pendingFileRef, len(c.Uploaded)) + copy(copied, c.Uploaded) + return sanitizePendingRefs(copied) + } + ids := nonEmptyIDs(c.FileIDs) + out := make([]pendingFileRef, 0, len(ids)) + for _, id := range ids { + out = append(out, pendingFileRef{ID: id}) + } + return out +} + +func (o *Orchestrator) appendPendingFiles(chatID, userID string, refs []pendingFileRef) { + refs = sanitizePendingRefs(refs) + if len(refs) == 0 { + return + } + key := pendingFileKey(chatID, userID) + o.pendingFilesMu.Lock() + defer o.pendingFilesMu.Unlock() + merged := append(o.pendingFiles[key], refs...) + o.pendingFiles[key] = sanitizePendingRefs(merged) +} + +func (o *Orchestrator) getPendingFiles(chatID, userID string) []pendingFileRef { + key := pendingFileKey(chatID, userID) + o.pendingFilesMu.Lock() + defer o.pendingFilesMu.Unlock() + snapshot := o.pendingFiles[key] + out := make([]pendingFileRef, len(snapshot)) + copy(out, snapshot) + return out +} + +func (o *Orchestrator) clearPendingFiles(chatID, userID string) { + key := pendingFileKey(chatID, userID) + o.pendingFilesMu.Lock() + defer o.pendingFilesMu.Unlock() + delete(o.pendingFiles, key) +} + +func pendingFileKey(chatID, userID string) string { + return strings.TrimSpace(chatID) + "::" + strings.TrimSpace(userID) +} + +func sanitizePendingRefs(refs []pendingFileRef) []pendingFileRef { + if len(refs) == 0 { + return nil + } + out := make([]pendingFileRef, 0, len(refs)) + seen := map[string]struct{}{} + for _, r := range refs { + id := strings.TrimSpace(r.ID) + if id == "" { + continue + } + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + r.ID = id + r.Name = strings.TrimSpace(r.Name) + r.MimeType = strings.TrimSpace(r.MimeType) + out = append(out, r) + } + return out +} + +func defaultIfEmpty(v, fallback string) string { + if strings.TrimSpace(v) == "" { + return fallback + } + return v +} + +// formatRelevantSkillsForPrompt 返回与当前用户问题最相关的技能内容。 +func (o *Orchestrator) formatSelectedSkillsForPrompt(userInput string, selected []knowledge.Skill) string { + skills := selected if len(skills) == 0 { - return "(none)" + skills = o.selectRelevantSkills(userInput, 4) + } + if len(skills) == 0 { + return "(none matched, tools are still globally available)" } return formatSkills(skills) } +func (o *Orchestrator) routeCapabilities(ctx context.Context, userInput string) capabilityRoutingResult { + fallback := capabilityRoutingResult{ + NeedSkills: true, + SelectedSkills: o.selectRelevantSkills(userInput, 4), + Reason: "router fallback: keyword matching", + UsedFallback: true, + } + + raw, err := o.llm.Generate(ctx, o.buildRouteSystemPrompt(), o.buildRouteUserPrompt(userInput)) + if err != nil { + if o.log != nil { + o.log.Warnf("capability router llm call failed err=%v", err) + } + return fallback + } + + decision, err := parseCapabilityRoute(raw) + if err != nil { + if o.log != nil { + o.log.Warnf("capability router parse failed err=%v raw=%q", err, raw) + } + return fallback + } + + resolvedTools := o.normalizeToolSelection(decision.SelectedTools) + resolved := capabilityRoutingResult{ + NeedSkills: decision.NeedSkills, + SelectedToolNames: resolvedTools, + Reason: strings.TrimSpace(decision.Reason), + } + + if resolved.NeedSkills { + skills := o.resolveSkillsByNames(decision.SelectedSkills, 4) + if len(skills) == 0 { + skills = o.selectRelevantSkills(userInput, 4) + resolved.UsedFallback = true + } + resolved.SelectedSkills = skills + } + + return resolved +} + +func (o *Orchestrator) buildRouteSystemPrompt() string { + return strings.Join([]string{ + "你是能力路由器（Router Agent）。", + "你的任务是：在不加载技能全文的前提下，仅根据工具摘要和技能摘要，判断本请求是否可以仅靠原子工具能力完成，还是需要加载技能详细说明。", + "输出必须且仅能是 JSON：", + "{", + " \"need_skills\": true 或 false,", + " \"selected_tools\": [\"tool_name\", ...],", + " \"selected_skills\": [\"skill_name\", ...],", + " \"reason\": \"简短路由理由\"", + "}", + "规则：", + "1) 优先原子工具能力。若可通过工具链路完成，need_skills=false。", + "2) 只有当工具能力不足以覆盖业务约束时，need_skills=true 并选择少量最相关技能。", + "3) selected_skills 仅填写技能名称（来自技能摘要）。", + "4) selected_tools 仅填写可用工具名。", + "5) 不要输出 JSON 之外内容。", + }, "\n") +} + +func (o *Orchestrator) buildRouteUserPrompt(userInput string) string { + return strings.Join([]string{ + "当前运行环境：", + formatRuntimeContextForPrompt(), + "", + "用户问题：", + userInput, + "", + "可用工具摘要：", + o.formatToolDoc(), + "", + "可用技能摘要：", + o.formatSkillSummariesForPrompt(), + "", + "请给出路由 JSON。", + }, "\n") +} + +func (o *Orchestrator) normalizeToolSelection(in []string) []string { + if len(in) == 0 { + return nil + } + allowed := map[string]struct{}{} + for _, t := range o.tools.List() { + allowed[strings.ToLower(strings.TrimSpace(t.Name()))] = struct{}{} + } + out := make([]string, 0, len(in)) + set := map[string]struct{}{} + for _, name := range in { + n := strings.ToLower(strings.TrimSpace(name)) + if n == "" { + continue + } + if _, ok := allowed[n]; !ok { + continue + } + if _, exists := set[n]; exists { + continue + } + set[n] = struct{}{} + out = append(out, n) + } + sort.Strings(out) + return out +} + +func (o *Orchestrator) resolveSkillsByNames(names []string, maxCount int) []knowledge.Skill { + if len(names) == 0 { + return nil + } + if maxCount <= 0 { + maxCount = 4 + } + all := o.getSkillsSnapshot() + idx := make(map[string]knowledge.Skill, len(all)) + for _, sk := range all { + key := strings.ToLower(strings.TrimSpace(sk.Name)) + if key != "" { + idx[key] = sk + } + } + out := make([]knowledge.Skill, 0, maxCount) + used := map[string]struct{}{} + for _, name := range names { + key := strings.ToLower(strings.TrimSpace(name)) + if key == "" { + continue + } + sk, ok := idx[key] + if !ok { + continue + } + if _, exists := used[key]; exists { + continue + } + used[key] = struct{}{} + out = append(out, sk) + if len(out) >= maxCount { + break + } + } + return out +} + +func formatRouteForPrompt(route capabilityRoutingResult) string { + b := strings.Builder{} + if route.UsedFallback { + b.WriteString("router_status: fallback\n") + } else { + b.WriteString("router_status: ok\n") + } + b.WriteString("need_skills: ") + b.WriteString(strconv.FormatBool(route.NeedSkills)) + b.WriteString("\n") + b.WriteString("selected_tools: ") + if len(route.SelectedToolNames) == 0 { + b.WriteString("(none)") + } else { + b.WriteString(strings.Join(route.SelectedToolNames, ", ")) + } + b.WriteString("\n") + b.WriteString("selected_skill_count: ") + b.WriteString(strconv.Itoa(len(route.SelectedSkills))) + b.WriteString("\n") + if strings.TrimSpace(route.Reason) != "" { + b.WriteString("reason: ") + b.WriteString(strings.TrimSpace(route.Reason)) + } + return strings.TrimSpace(b.String()) +} + +func (o *Orchestrator) selectRelevantSkills(userInput string, maxCount int) []knowledge.Skill { + if maxCount <= 0 { + maxCount = 4 + } + query := strings.TrimSpace(strings.ToLower(userInput)) + all := o.getSkillsSnapshot() + if query == "" || len(all) <= maxCount { + return all + } + + queryTokens := buildQueryTokens(query) + type item struct { + skill knowledge.Skill + score int + } + ranked := make([]item, 0, len(all)) + + for _, sk := range all { + hay := strings.ToLower(sk.Name + "\n" + clipForScoring(sk.Content, 1800)) + score := 0 + if strings.Contains(hay, query) { + score += 8 + } + for _, tk := range queryTokens { + if strings.Contains(hay, tk) { + score++ + } + } + if score == 0 { + continue + } + ranked = append(ranked, item{skill: sk, score: score}) + } + + if len(ranked) == 0 { + return nil + } + + sort.Slice(ranked, func(i, j int) bool { + if ranked[i].score == ranked[j].score { + return strings.ToLower(strings.TrimSpace(ranked[i].skill.Name)) < strings.ToLower(strings.TrimSpace(ranked[j].skill.Name)) + } + return ranked[i].score > ranked[j].score + }) + + if len(ranked) > maxCount { + ranked = ranked[:maxCount] + } + out := make([]knowledge.Skill, 0, len(ranked)) + for _, r := range ranked { + out = append(out, r.skill) + } + return out +} + +func buildQueryTokens(query string) []string { + set := map[string]struct{}{} + collectToken := func(t string) { + t = strings.TrimSpace(t) + if len([]rune(t)) < 2 { + return + } + set[t] = struct{}{} + } + + for _, part := range strings.FieldsFunc(query, func(r rune) bool { + if r >= 'a' && r <= 'z' { + return false + } + if r >= '0' && r <= '9' { + return false + } + if r >= 0x4e00 && r <= 0x9fff { + return false + } + return true + }) { + collectToken(part) + } + + // 针对中文无空格输入，补充 2-gram 提升匹配命中率。 + runes := []rune(query) + for i := 0; i+1 < len(runes); i++ { + r1 := runes[i] + r2 := runes[i+1] + if (r1 >= 0x4e00 && r1 <= 0x9fff) && (r2 >= 0x4e00 && r2 <= 0x9fff) { + collectToken(string([]rune{r1, r2})) + } + } + + out := make([]string, 0, len(set)) + for tk := range set { + out = append(out, tk) + } + sort.Strings(out) + return out +} + +func clipForScoring(s string, maxRunes int) string { + if maxRunes <= 0 { + maxRunes = 1800 + } + r := []rune(s) + if len(r) <= maxRunes { + return s + } + return string(r[:maxRunes]) +} + +func formatRuntimeContextForPrompt() string { + goos := strings.TrimSpace(strings.ToLower(runtime.GOOS)) + if goos == "" { + goos = "unknown" + } + return "当前运行系统 GOOS=" + goos + "。请优先使用与该系统一致的策略。仅当用户明确要求时，才采用其他系统（如 Windows）的专用流程。" +} + // emitCapabilityGap 处理能力缺口信息埋点或者通过 AI 自动创建生成相应缺失技能的逻辑 func (o *Orchestrator) emitCapabilityGap(chatID, userID, intent, reason string) { if !o.enableCapabilityGap { diff --git a/internal/agent/orchestrator_skill_selection_test.go b/internal/agent/orchestrator_skill_selection_test.go new file mode 100644 index 0000000..29cf1ec --- /dev/null +++ b/internal/agent/orchestrator_skill_selection_test.go @@ -0,0 +1,48 @@ +package agent + +import ( + "runtime" + "strings" + "testing" + + "laodingbot/internal/knowledge" +) + +func TestBuildQueryTokensIncludesChineseBigrams(t *testing.T) { + tokens := buildQueryTokens("请执行命令并查看文件") + joined := strings.Join(tokens, ",") + if !strings.Contains(joined, "命令") { + t.Fatalf("expected token contains 命令, got: %v", tokens) + } + if !strings.Contains(joined, "文件") { + t.Fatalf("expected token contains 文件, got: %v", tokens) + } +} + +func TestSelectRelevantSkillsPrefersMatchingSkill(t *testing.T) { + o := &Orchestrator{ + skills: []knowledge.Skill{ + {Name: "文件系统查询专家", Content: "适用于目录、文件、路径、命令执行等场景"}, + {Name: "天气查询", Content: "用于天气和空气质量查询"}, + {Name: "日程助手", Content: "用于日程管理"}, + }, + } + + selected := o.selectRelevantSkills("帮我执行命令查看某个文件", 2) + if len(selected) == 0 { + t.Fatal("expected non-empty selected skills") + } + if selected[0].Name != "文件系统查询专家" { + t.Fatalf("expected top skill 文件系统查询专家, got: %s", selected[0].Name) + } + if len(selected) > 2 { + t.Fatalf("expected at most 2 skills, got: %d", len(selected)) + } +} + +func TestFormatRuntimeContextForPromptIncludesGOOS(t *testing.T) { + doc := formatRuntimeContextForPrompt() + if !strings.Contains(strings.ToLower(doc), strings.ToLower(runtime.GOOS)) { + t.Fatalf("expected runtime context contains GOOS=%s, got: %s", runtime.GOOS, doc) + } +} diff --git a/internal/agent/router_parser.go b/internal/agent/router_parser.go new file mode 100644 index 0000000..dbbffc4 --- /dev/null +++ b/internal/agent/router_parser.go @@ -0,0 +1,31 @@ +package agent + +import ( + "encoding/json" + "fmt" + "strings" +) + +type capabilityRouteDecision struct { + NeedSkills bool `json:"need_skills"` + SelectedTools []string `json:"selected_tools"` + SelectedSkills []string `json:"selected_skills"` + Reason string `json:"reason"` +} + +func parseCapabilityRoute(raw string) (capabilityRouteDecision, error) { + raw = normalizeJSON(raw) + start := strings.Index(raw, "{") + end := strings.LastIndex(raw, "}") + if start < 0 || end < start { + return capabilityRouteDecision{}, fmt.Errorf("no json object found") + } + raw = raw[start : end+1] + + var out capabilityRouteDecision + if err := json.Unmarshal([]byte(raw), &out); err != nil { + return capabilityRouteDecision{}, err + } + out.Reason = strings.TrimSpace(out.Reason) + return out, nil +} diff --git a/internal/agent/router_parser_test.go b/internal/agent/router_parser_test.go new file mode 100644 index 0000000..e302a08 --- /dev/null +++ b/internal/agent/router_parser_test.go @@ -0,0 +1,34 @@ +package agent + +import "testing" + +func TestParseCapabilityRoute(t *testing.T) { + raw := `{"need_skills":true,"selected_tools":["shell"],"selected_skills":["文件系统查询专家"],"reason":"需要技能约束"}` + out, err := parseCapabilityRoute(raw) + if err != nil { + t.Fatalf("parseCapabilityRoute error: %v", err) + } + if !out.NeedSkills { + t.Fatal("expected need_skills=true") + } + if len(out.SelectedTools) != 1 || out.SelectedTools[0] != "shell" { + t.Fatalf("unexpected selected_tools: %#v", out.SelectedTools) + } + if len(out.SelectedSkills) != 1 || out.SelectedSkills[0] != "文件系统查询专家" { + t.Fatalf("unexpected selected_skills: %#v", out.SelectedSkills) + } +} + +func TestParseCapabilityRouteCodeFence(t *testing.T) { + raw := "```json\n{\"need_skills\":false,\"selected_tools\":[\"file\",\"shell\"],\"selected_skills\":[],\"reason\":\"工具足够\"}\n```" + out, err := parseCapabilityRoute(raw) + if err != nil { + t.Fatalf("parseCapabilityRoute error: %v", err) + } + if out.NeedSkills { + t.Fatal("expected need_skills=false") + } + if len(out.SelectedTools) != 2 { + t.Fatalf("unexpected selected_tools len: %d", len(out.SelectedTools)) + } +} diff --git a/internal/llm/client.go b/internal/llm/client.go index 4a21845..bd435f1 100644 --- a/internal/llm/client.go +++ b/internal/llm/client.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "io" + "mime/multipart" "net/http" "strings" "time" @@ -18,6 +19,20 @@ type Client interface { Generate(ctx context.Context, systemPrompt, userPrompt string) (string, error) } +type FileChatClient interface { + GenerateWithFiles(ctx context.Context, systemPrompt, userPrompt string, fileIDs []string) (string, error) +} + +type FileUploader interface { + UploadFile(ctx context.Context, file InputFile, purpose string) (string, error) +} + +type InputFile struct { + FileName string + MimeType string + Content []byte +} + type OpenAICompatibleClient struct { baseURL string apiKey string @@ -43,27 +58,64 @@ type chatRequest struct { type chatMessage struct { Role string `json:"role"` - Content string `json:"content"` + Content any `json:"content"` +} + +type chatContentPart struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + FileID string `json:"file_id,omitempty"` } type chatResponse struct { Choices []struct { - Message chatMessage `json:"message"` + Message struct { + Role string `json:"role"` + Content string `json:"content"` + } `json:"message"` } `json:"choices"` Error *struct { Message string `json:"message"` } `json:"error,omitempty"` } +type fileUploadResponse struct { + ID string `json:"id"` + Bytes int64 `json:"bytes,omitempty"` + CreatedAt int64 `json:"created_at,omitempty"` + Filename string `json:"filename,omitempty"` + Object string `json:"object,omitempty"` + Purpose string `json:"purpose,omitempty"` + Code int `json:"code,omitempty"` + Message string `json:"message,omitempty"` + Status any `json:"status,omitempty"` + StatusDetails any `json:"status_details,omitempty"` + Data *struct { + ID string `json:"id"` + } `json:"data,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + func (c *OpenAICompatibleClient) Generate(ctx context.Context, systemPrompt, userPrompt string) (string, error) { + return c.generateInternal(ctx, systemPrompt, userPrompt, nil) +} + +func (c *OpenAICompatibleClient) GenerateWithFiles(ctx context.Context, systemPrompt, userPrompt string, fileIDs []string) (string, error) { + return c.generateInternal(ctx, systemPrompt, userPrompt, fileIDs) +} + +func (c *OpenAICompatibleClient) generateInternal(ctx context.Context, systemPrompt, userPrompt string, fileIDs []string) (string, error) { if c.log != nil { - c.log.Debugf("llm request start model=%s system_len=%d user_len=%d", c.model, len(systemPrompt), len(userPrompt)) + c.log.Debugf("llm request start model=%s system_len=%d user_len=%d file_count=%d", c.model, len(systemPrompt), len(userPrompt), len(fileIDs)) } + userContent := buildUserContent(userPrompt, fileIDs) body := chatRequest{ Model: c.model, Messages: []chatMessage{ {Role: "system", Content: systemPrompt}, - {Role: "user", Content: userPrompt}, + {Role: "user", Content: userContent}, }, } b, err := json.Marshal(body) @@ -132,3 +184,144 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, systemPrompt, use return out.Choices[0].Message.Content, nil } + +func buildUserContent(userPrompt string, fileIDs []string) any { + trimmedPrompt := strings.TrimSpace(userPrompt) + if len(fileIDs) == 0 { + return userPrompt + } + + parts := make([]chatContentPart, 0, len(fileIDs)+1) + if trimmedPrompt != "" { + parts = append(parts, chatContentPart{Type: "text", Text: userPrompt}) + } + for _, id := range fileIDs { + id = strings.TrimSpace(id) + if id == "" { + continue + } + parts = append(parts, chatContentPart{Type: "file", FileID: id}) + } + if len(parts) == 0 { + return userPrompt + } + return parts +} + +func (c *OpenAICompatibleClient) UploadFile(ctx context.Context, file InputFile, purpose string) (string, error) { + if strings.TrimSpace(file.FileName) == "" { + return "", fmt.Errorf("empty file name") + } + if len(file.Content) == 0 { + return "", fmt.Errorf("empty file content") + } + purpose = strings.TrimSpace(purpose) + purposes := []string{} + if purpose != "" { + purposes = append(purposes, purpose) + } + // Provider compatibility fallback order. + purposes = appendIfMissing(purposes, "file-extract") + purposes = appendIfMissing(purposes, "batch") + + var lastErr error + for _, p := range purposes { + fileID, err := c.uploadFileOnce(ctx, file, p) + if err == nil { + return fileID, nil + } + lastErr = err + if c.log != nil { + c.log.Warnf("llm file upload failed purpose=%s err=%v", p, err) + } + } + if lastErr == nil { + lastErr = fmt.Errorf("llm file upload failed: no purpose tried") + } + return "", lastErr +} + +func (c *OpenAICompatibleClient) uploadFileOnce(ctx context.Context, file InputFile, purpose string) (string, error) { + + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + if err := writer.WriteField("purpose", purpose); err != nil { + return "", err + } + part, err := writer.CreateFormFile("file", file.FileName) + if err != nil { + return "", err + } + if _, err := part.Write(file.Content); err != nil { + return "", err + } + if err := writer.Close(); err != nil { + return "", err + } + + url := strings.TrimRight(c.baseURL, "/") + "/files" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, body) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Authorization", "Bearer "+c.apiKey) + + resp, err := c.http.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + raw, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + var out fileUploadResponse + if err := json.Unmarshal(raw, &out); err != nil { + return "", fmt.Errorf("llm file upload response decode failed: %w body=%s", err, clipForError(raw)) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + if strings.TrimSpace(out.Message) != "" { + return "", fmt.Errorf("llm file upload error: %s", out.Message) + } + if out.Error != nil && out.Error.Message != "" { + return "", fmt.Errorf("llm file upload error: %s", out.Error.Message) + } + return "", fmt.Errorf("llm file upload status: %d body=%s", resp.StatusCode, clipForError(raw)) + } + fileID := strings.TrimSpace(out.ID) + if fileID == "" && out.Data != nil { + fileID = strings.TrimSpace(out.Data.ID) + } + if fileID == "" { + return "", fmt.Errorf("llm file upload returned empty file id body=%s", clipForError(raw)) + } + if c.log != nil { + c.log.Infof("llm file uploaded name=%s size=%d file_id=%s purpose=%s status=%v", file.FileName, len(file.Content), fileID, purpose, out.Status) + } + return fileID, nil +} + +func clipForError(raw []byte) string { + s := strings.TrimSpace(string(raw)) + const max = 400 + if len(s) <= max { + return s + } + return s[:max] + "...(truncated)" +} + +func appendIfMissing(items []string, value string) []string { + value = strings.TrimSpace(value) + if value == "" { + return items + } + for _, it := range items { + if strings.EqualFold(strings.TrimSpace(it), value) { + return items + } + } + return append(items, value) +} diff --git a/internal/toolhost/runtime.go b/internal/toolhost/runtime.go index a1c765c..4cac07b 100644 --- a/internal/toolhost/runtime.go +++ b/internal/toolhost/runtime.go @@ -9,6 +9,7 @@ import ( "laodingbot/internal/logger" "laodingbot/internal/tools" "laodingbot/tools/fileoperation" + "laodingbot/tools/git" "laodingbot/tools/shell" "laodingbot/tools/websearch" ) @@ -16,6 +17,7 @@ import ( func RunChild(ctx context.Context, cfg config.Config, log *logger.Logger) error { var registryLog *logger.Logger var fileLog *logger.Logger + var gitLog *logger.Logger var shellLog *logger.Logger var searchLog *logger.Logger var serverLog *logger.Logger @@ -23,12 +25,19 @@ func RunChild(ctx context.Context, cfg config.Config, log *logger.Logger) error log.Infof("toolhost child starting") registryLog = log.WithComponent("toolhost.registry") fileLog = log.WithComponent("toolhost.file") + gitLog = log.WithComponent("toolhost.git") shellLog = log.WithComponent("toolhost.shell") searchLog = log.WithComponent("toolhost.websearch") serverLog = log.WithComponent("toolhost.server") } registry := tools.NewRegistry(registryLog) registry.Register(fileoperation.New(cfg.Security.AllowedDirs, cfg.ToolOutputMaxChars, fileLog)) + registry.Register(git.New( + cfg.Security.WorkDir, + time.Duration(cfg.ToolCallTimeoutSec)*time.Second, + cfg.ToolOutputMaxChars, + gitLog, + )) registry.Register(shell.New( cfg.Security.AllowedCommands, cfg.Security.WorkDir, diff --git a/internal/transport/feishu/bot.go b/internal/transport/feishu/bot.go index 52ba5bb..9e1080f 100644 --- a/internal/transport/feishu/bot.go +++ b/internal/transport/feishu/bot.go @@ -4,6 +4,10 @@ import ( "context" "encoding/json" "fmt" + "io" + "mime" + "os" + "path/filepath" "strings" "sync" "time" @@ -34,9 +38,17 @@ type IncomingMessage struct { MessageID string ChatID string UserID string + MsgType string Text string + FileName string + FileKey string + FileMime string + FileBytes []byte + FilePath string } +const maxFeishuFileBytes = 20 * 1024 * 1024 + func NewBot(appID, appSecret, verifyToken, _ string, _ string, log *logger.Logger) (*Bot, error) { if appID == "" || appSecret == "" { return nil, fmt.Errorf("empty feishu app credentials") @@ -66,7 +78,7 @@ func (b *Bot) Run(ctx context.Context, handler func(context.Context, IncomingMes incoming, ok := parseIncoming(event) if !ok { if b.log != nil { - b.log.Debugf("skip non-text or invalid feishu event") + b.log.Debugf("skip unsupported or invalid feishu event") } return nil } @@ -76,8 +88,11 @@ func (b *Bot) Run(ctx context.Context, handler func(context.Context, IncomingMes } return nil } + if incoming.MsgType == "file" { + b.enrichFileIncoming(evtCtx, &incoming) + } if b.log != nil { - b.log.Infof("feishu message received message_id=%s chat_id=%s user_id=%s text=%s", incoming.MessageID, incoming.ChatID, incoming.UserID, incoming.Text) + b.log.Infof("feishu message received message_id=%s chat_id=%s user_id=%s msg_type=%s text=%s", incoming.MessageID, incoming.ChatID, incoming.UserID, incoming.MsgType, incoming.Text) } reply, err := handler(evtCtx, incoming) if err != nil { @@ -159,6 +174,175 @@ func extractText(content string) (string, error) { return parsed.Text, nil } +func extractFileMeta(content string) (fileName string, fileKey string, err error) { + var parsed map[string]any + if err := json.Unmarshal([]byte(content), &parsed); err != nil { + return "", "", err + } + + readString := func(keys ...string) string { + for _, key := range keys { + if v, ok := parsed[key]; ok { + s, ok := v.(string) + if ok { + trimmed := strings.TrimSpace(s) + if trimmed != "" { + return trimmed + } + } + } + } + return "" + } + + fileName = readString("file_name", "fileName", "name", "filename") + fileKey = readString("file_key", "fileKey", "key") + return fileName, fileKey, nil +} + +func buildFileRecognitionText(fileName, fileKey string) string { + if strings.TrimSpace(fileName) == "" { + fileName = "(unknown)" + } + if strings.TrimSpace(fileKey) == "" { + fileKey = "(unknown)" + } + + return strings.Join([]string{ + "用户发送了一条飞书文件消息。", + "文件名: " + fileName, + "文件Key: " + fileKey, + "系统将先上传该文件到 LLM Provider，再由模型完成文档解析。若上传失败，本次请求将直接中止。", + }, "\n") +} + +func (b *Bot) enrichFileIncoming(ctx context.Context, incoming *IncomingMessage) { + if incoming == nil { + return + } + if strings.TrimSpace(incoming.MessageID) == "" || strings.TrimSpace(incoming.FileKey) == "" { + incoming.Text = buildFileRecognitionText(incoming.FileName, incoming.FileKey) + incoming.Text += "\n\n未找到完整 file_key 或 message_id，暂时无法下载文件内容。" + return + } + + content, fileName, err := b.downloadFileContent(ctx, incoming.MessageID, incoming.FileKey) + if err != nil { + if b.log != nil { + b.log.Warnf("feishu download file content failed message_id=%s file_key=%s err=%v", incoming.MessageID, incoming.FileKey, err) + } + incoming.Text = buildFileRecognitionText(incoming.FileName, incoming.FileKey) + incoming.Text += "\n\n文件下载失败: " + err.Error() + return + } + if strings.TrimSpace(fileName) != "" { + incoming.FileName = fileName + } + incoming.FileBytes = content + incoming.FileMime = detectMimeByName(incoming.FileName) + localPath, saveErr := saveIncomingFile("files", incoming.FileName, incoming.FileBytes) + if saveErr != nil { + if b.log != nil { + b.log.Warnf("save incoming feishu file failed name=%s err=%v", incoming.FileName, saveErr) + } + incoming.Text = buildFileRecognitionText(incoming.FileName, incoming.FileKey) + incoming.Text += "\n\n文件已下载但本地保存失败: " + saveErr.Error() + return + } + incoming.FilePath = localPath + incoming.Text = buildFileRecognitionText(incoming.FileName, incoming.FileKey) + incoming.Text += fmt.Sprintf("\n\n文件已下载并保存到本地，路径=%s，大小=%d bytes，mime=%s。", incoming.FilePath, len(content), incoming.FileMime) +} + +func (b *Bot) downloadFileContent(ctx context.Context, messageID, fileKey string) ([]byte, string, error) { + req := larkim.NewGetMessageResourceReqBuilder(). + MessageId(messageID). + FileKey(fileKey). + Type("file"). + Build() + + resp, err := b.apiClient.Im.MessageResource.Get(ctx, req) + if err != nil { + return nil, "", err + } + if resp == nil || resp.File == nil { + if resp != nil { + return nil, "", fmt.Errorf("empty file stream code=%d msg=%s", resp.Code, resp.Msg) + } + return nil, "", fmt.Errorf("empty file stream") + } + + bts, err := io.ReadAll(resp.File) + if err != nil { + return nil, "", err + } + if len(bts) > maxFeishuFileBytes { + return nil, "", fmt.Errorf("file too large: %d bytes, max=%d", len(bts), maxFeishuFileBytes) + } + return bts, strings.TrimSpace(resp.FileName), nil +} + +func detectMimeByName(fileName string) string { + ext := strings.ToLower(strings.TrimSpace(filepath.Ext(fileName))) + if ext == "" { + return "application/octet-stream" + } + m := strings.TrimSpace(mime.TypeByExtension(ext)) + if m == "" { + return "application/octet-stream" + } + return m +} + +func saveIncomingFile(baseDir, fileName string, content []byte) (string, error) { + if len(content) == 0 { + return "", fmt.Errorf("empty file content") + } + if strings.TrimSpace(baseDir) == "" { + baseDir = "files" + } + if err := os.MkdirAll(baseDir, 0o755); err != nil { + return "", err + } + safeName := sanitizeFileName(fileName) + if safeName == "" { + safeName = "upload.bin" + } + finalName := fmt.Sprintf("%d_%s", time.Now().UnixNano(), safeName) + target := filepath.Join(baseDir, finalName) + if err := os.WriteFile(target, content, 0o644); err != nil { + return "", err + } + abs, err := filepath.Abs(target) + if err != nil { + return target, nil + } + return abs, nil +} + +func sanitizeFileName(fileName string) string { + name := strings.TrimSpace(filepath.Base(fileName)) + if name == "" || name == "." || name == ".." { + return "" + } + var b strings.Builder + for _, r := range name { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' { + b.WriteRune(r) + continue + } + b.WriteByte('_') + } + out := strings.TrimSpace(b.String()) + if out == "" || out == "." || out == ".." { + return "" + } + if strings.HasPrefix(out, ".") { + out = "file" + out + } + return out +} + func parseIncoming(event *larkim.P2MessageReceiveV1) (IncomingMessage, bool) { if event == nil || event.Event == nil || event.Event.Message == nil || event.Event.Sender == nil || event.Event.Sender.SenderId == nil { return IncomingMessage{}, false @@ -168,12 +352,11 @@ func parseIncoming(event *larkim.P2MessageReceiveV1) (IncomingMessage, bool) { } msg := event.Event.Message - if msg.MessageType == nil || *msg.MessageType != "text" || msg.ChatId == nil || msg.Content == nil || msg.MessageId == nil { + if msg.MessageType == nil || msg.ChatId == nil || msg.Content == nil || msg.MessageId == nil { return IncomingMessage{}, false } - - text, err := extractText(*msg.Content) - if err != nil { + msgType := strings.TrimSpace(*msg.MessageType) + if msgType == "" { return IncomingMessage{}, false } @@ -186,12 +369,33 @@ func parseIncoming(event *larkim.P2MessageReceiveV1) (IncomingMessage, bool) { userID = *event.Event.Sender.SenderId.UnionId } - return IncomingMessage{ + incoming := IncomingMessage{ MessageID: *msg.MessageId, ChatID: *msg.ChatId, UserID: userID, - Text: text, - }, true + MsgType: msgType, + } + + switch msgType { + case "text": + text, err := extractText(*msg.Content) + if err != nil { + return IncomingMessage{}, false + } + incoming.Text = text + return incoming, true + case "file": + fileName, fileKey, err := extractFileMeta(*msg.Content) + if err != nil { + return IncomingMessage{}, false + } + incoming.FileName = fileName + incoming.FileKey = fileKey + incoming.Text = buildFileRecognitionText(fileName, fileKey) + return incoming, true + default: + return IncomingMessage{}, false + } } func (b *Bot) shouldProcessMessage(messageID string) bool { diff --git a/internal/transport/feishu/bot_test.go b/internal/transport/feishu/bot_test.go new file mode 100644 index 0000000..ccc74fc --- /dev/null +++ b/internal/transport/feishu/bot_test.go @@ -0,0 +1,138 @@ +package feishu + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + larkim "github.com/larksuite/oapi-sdk-go/v3/service/im/v1" +) + +func mustEventFromJSON(t *testing.T, raw string) *larkim.P2MessageReceiveV1 { + t.Helper() + var evt larkim.P2MessageReceiveV1 + if err := json.Unmarshal([]byte(raw), &evt); err != nil { + t.Fatalf("unmarshal event json failed: %v", err) + } + return &evt +} + +func TestParseIncomingText(t *testing.T) { + evt := mustEventFromJSON(t, `{ + "event": { + "message": { + "message_id": "msg_text_1", + "chat_id": "chat_1", + "message_type": "text", + "content": "{\"text\":\"你好\"}" + }, + "sender": { + "sender_type": "user", + "sender_id": {"open_id": "u_open_1"} + } + } + }`) + + in, ok := parseIncoming(evt) + if !ok { + t.Fatal("expected text message parse success") + } + if in.MsgType != "text" { + t.Fatalf("expected msg type text, got %s", in.MsgType) + } + if in.Text != "你好" { + t.Fatalf("unexpected text: %q", in.Text) + } +} + +func TestParseIncomingFile(t *testing.T) { + evt := mustEventFromJSON(t, `{ + "event": { + "message": { + "message_id": "msg_file_1", + "chat_id": "chat_1", + "message_type": "file", + "content": "{\"file_key\":\"file_key_123\",\"file_name\":\"report.pdf\"}" + }, + "sender": { + "sender_type": "user", + "sender_id": {"user_id": "u_id_1"} + } + } + }`) + + in, ok := parseIncoming(evt) + if !ok { + t.Fatal("expected file message parse success") + } + if in.MsgType != "file" { + t.Fatalf("expected msg type file, got %s", in.MsgType) + } + if in.FileName != "report.pdf" || in.FileKey != "file_key_123" { + t.Fatalf("unexpected file meta: name=%q key=%q", in.FileName, in.FileKey) + } + if !strings.Contains(in.Text, "飞书文件消息") { + t.Fatalf("expected synthesized text mentions file message, got: %q", in.Text) + } +} + +func TestParseIncomingUnsupportedType(t *testing.T) { + evt := mustEventFromJSON(t, `{ + "event": { + "message": { + "message_id": "msg_image_1", + "chat_id": "chat_1", + "message_type": "image", + "content": "{\"image_key\":\"img_1\"}" + }, + "sender": { + "sender_type": "user", + "sender_id": {"open_id": "u_open_1"} + } + } + }`) + + _, ok := parseIncoming(evt) + if ok { + t.Fatal("expected unsupported message type rejected") + } +} + +func TestDetectMimeByName(t *testing.T) { + if got := detectMimeByName("report.pdf"); !strings.Contains(got, "pdf") { + t.Fatalf("expected pdf mime, got: %s", got) + } + if got := detectMimeByName("unknown.custom"); got != "application/octet-stream" { + t.Fatalf("expected octet-stream fallback, got: %s", got) + } +} + +func TestSaveIncomingFile(t *testing.T) { + dir := t.TempDir() + path, err := saveIncomingFile(dir, "report.pdf", []byte("hello")) + if err != nil { + t.Fatalf("saveIncomingFile error: %v", err) + } + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read saved file failed: %v", err) + } + if string(b) != "hello" { + t.Fatalf("unexpected saved content: %q", string(b)) + } + if filepath.Ext(path) != ".pdf" { + t.Fatalf("expected .pdf extension, got: %s", path) + } +} + +func TestSanitizeFileName(t *testing.T) { + got := sanitizeFileName("../bad path/测 试?.pdf") + if strings.Contains(got, "/") || strings.Contains(got, "\\") { + t.Fatalf("expected sanitized basename only, got: %q", got) + } + if !strings.HasSuffix(got, ".pdf") { + t.Fatalf("expected .pdf suffix, got: %q", got) + } +} diff --git a/skills/architecture-wbs-breakdown/skill.md b/skills/architecture-wbs-breakdown/skill.md new file mode 100644 index 0000000..8474be1 --- /dev/null +++ b/skills/architecture-wbs-breakdown/skill.md @@ -0,0 +1,29 @@ +--- +name: architecture-wbs-breakdown +description: 作为架构师，将复杂需求拆解为多团队 WBS 详细文档，并自动提交至 Git 仓库 +--- + +# 复杂系统需求拆解与多团队 WBS 规划 + +## 使用场景 +当接收到业务需求文档，需要进行工程化拆解并进行版本化归档时使用，例如： + +- 将单体需求拆解为适合多团队并行开发的 SAFe/PI 规划 WBS +- 界定系统中不同子系统（或软硬件团队）的交互边界与契约 +- 将生成的拆解报告自动化落地并推送到远程代码仓库 + +## 架构拆分与团队规划 +- 提取核心业务流程与全局非功能性需求 (NFR) +- 运用限界上下文划分独立演进的子系统，明确交互契约（API、RPC、ICD 等） +- 为每个子系统配置专属开发团队，定义角色配置（SM, PO, 开发, 测试） + +## WBS 拆解与输出约束 +- 按 Epic -> Feature -> Story/Task 的敏捷层级，输出结构清晰、详细的 Markdown 文档 +- 明确跨团队的交付物契约，梳理 Feature 依赖关系以识别集成风险点 + +## 文档归档与版本控制 (工作流执行) +文档生成完毕后，必须触发以下自动化流程： +- **文件保存**：将生成的 Markdown 文档以合适的命名（如 `wbs_规划_[日期].md`）写入本地的 `new_floder` 文件夹中。 +- **版本追踪**：使用 Git 管理该文件夹，执行 `git add .`（或指定该 md 文件）。 +- **提交变更**：生成准确的提交信息（例如 `docs: add WBS breakdown for [项目/需求简述]`），执行 `git commit`。 +- **远程同步**：将本地提交推送到远程 Git 仓库，执行 `git push`。 \ No newline at end of file diff --git a/tools/git/git.go b/tools/git/git.go new file mode 100644 index 0000000..024026e --- /dev/null +++ b/tools/git/git.go @@ -0,0 +1,149 @@ +package git + +import ( + "context" + "fmt" + "os/exec" + "path/filepath" + "strings" + "time" + + "laodingbot/internal/logger" +) + +// Tool provides common Git operations through a safe command whitelist. +type Tool struct { + repoDir string + allowedSubcmds map[string]struct{} + timeout time.Duration + maxOutputChars int + log *logger.Logger +} + +// New creates a new git tool. +func New(repoDir string, timeout time.Duration, maxOutputChars int, log *logger.Logger) *Tool { + absRepo, err := filepath.Abs(strings.TrimSpace(repoDir)) + if err != nil { + absRepo = strings.TrimSpace(repoDir) + } + if timeout <= 0 { + timeout = 20 * time.Second + } + if maxOutputChars <= 0 { + maxOutputChars = 4000 + } + + allowed := map[string]struct{}{ + "status": {}, + "log": {}, + "show": {}, + "diff": {}, + "branch": {}, + "checkout": {}, + "switch": {}, + "restore": {}, + "add": {}, + "commit": {}, + "reset": {}, + "revert": {}, + "merge": {}, + "rebase": {}, + "cherry-pick": {}, + "fetch": {}, + "pull": {}, + "push": {}, + "remote": {}, + "tag": {}, + "stash": {}, + "blame": {}, + "rev-parse": {}, + } + + if log != nil { + log.Infof("git tool initialized repo_dir=%s timeout=%s max_output_chars=%d", absRepo, timeout, maxOutputChars) + } + + return &Tool{ + repoDir: absRepo, + allowedSubcmds: allowed, + timeout: timeout, + maxOutputChars: maxOutputChars, + log: log, + } +} + +func (t *Tool) Name() string { return "git" } + +func (t *Tool) Description() string { + return "Run common git commands inside repository. Input examples: status --short | log --oneline -n 10 | add . | commit -m fix | branch" +} + +func (t *Tool) Call(ctx context.Context, input string) (string, error) { + trimmed := strings.TrimSpace(input) + if trimmed == "" { + if t.log != nil { + t.log.Warnf("git tool rejected empty command") + } + return "", fmt.Errorf("empty git command") + } + + parts := strings.Fields(trimmed) + if len(parts) == 0 { + return "", fmt.Errorf("empty git command") + } + + if strings.EqualFold(parts[0], "git") { + parts = parts[1:] + } + if len(parts) == 0 { + return "", fmt.Errorf("missing git subcommand") + } + + subcmd := strings.ToLower(parts[0]) + if _, ok := t.allowedSubcmds[subcmd]; !ok { + if t.log != nil { + t.log.Warnf("git tool rejected subcommand=%s", subcmd) + } + return "", fmt.Errorf("unsupported git subcommand: %s", subcmd) + } + + runCtx, cancel := context.WithTimeout(ctx, t.timeout) + defer cancel() + + args := append([]string{subcmd}, parts[1:]...) + if t.log != nil { + t.log.Infof("git command start subcommand=%s args=%d full=%q", subcmd, len(parts)-1, strings.Join(args, " ")) + } + + cmd := exec.CommandContext(runCtx, "git", args...) + cmd.Dir = t.repoDir + cmd.Env = append(cmd.Environ(), + "GIT_TERMINAL_PROMPT=0", + "GIT_PAGER=cat", + "GIT_EDITOR=true", + ) + + out, err := cmd.CombinedOutput() + outText := strings.TrimSpace(string(out)) + if len(outText) > t.maxOutputChars { + outText = outText[:t.maxOutputChars] + } + + if err != nil { + if t.log != nil { + t.log.Errorf("git command failed subcommand=%s err=%v output=%q", subcmd, err, outText) + } + if outText == "" { + return "", err + } + return outText, err + } + + if t.log != nil { + t.log.Infof("git command success subcommand=%s output_bytes=%d", subcmd, len(out)) + } + if outText == "" { + return "ok", nil + } + return outText, nil +} diff --git a/tools/git/git_test.go b/tools/git/git_test.go new file mode 100644 index 0000000..c556bca --- /dev/null +++ b/tools/git/git_test.go @@ -0,0 +1,129 @@ +package git + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +func TestCallRejectsEmptyCommand(t *testing.T) { + tool := New(t.TempDir(), 3*time.Second, 4000, nil) + _, err := tool.Call(context.Background(), " ") + if err == nil { + t.Fatal("expected error for empty command") + } +} + +func TestCallRejectsUnsupportedSubcommand(t *testing.T) { + repo := initTestRepo(t) + tool := New(repo, 5*time.Second, 4000, nil) + + _, err := tool.Call(context.Background(), "bisect start") + if err == nil { + t.Fatal("expected unsupported subcommand error") + } + if !strings.Contains(err.Error(), "unsupported git subcommand") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestStatusAndLog(t *testing.T) { + repo := initTestRepo(t) + tool := New(repo, 5*time.Second, 4000, nil) + + status, err := tool.Call(context.Background(), "status --short") + if err != nil { + t.Fatalf("status failed: %v", err) + } + if strings.TrimSpace(status) != "" && status != "ok" { + t.Fatalf("expected clean status output, got: %q", status) + } + + logOut, err := tool.Call(context.Background(), "git log --oneline -n 1") + if err != nil { + t.Fatalf("log failed: %v", err) + } + if !strings.Contains(logOut, "initial commit") { + t.Fatalf("expected initial commit in log output, got: %q", logOut) + } +} + +func TestAddCommitAndDiff(t *testing.T) { + repo := initTestRepo(t) + tool := New(repo, 8*time.Second, 8000, nil) + + file := filepath.Join(repo, "note.txt") + if err := os.WriteFile(file, []byte("v1\n"), 0o644); err != nil { + t.Fatalf("write file failed: %v", err) + } + + _, err := tool.Call(context.Background(), "add note.txt") + if err != nil { + t.Fatalf("add failed: %v", err) + } + + _, err = tool.Call(context.Background(), "commit -m add-note") + if err != nil { + t.Fatalf("commit failed: %v", err) + } + + if err := os.WriteFile(file, []byte("v2\n"), 0o644); err != nil { + t.Fatalf("rewrite file failed: %v", err) + } + + diffOut, err := tool.Call(context.Background(), "diff -- note.txt") + if err != nil { + t.Fatalf("diff failed: %v", err) + } + if !strings.Contains(diffOut, "-v1") || !strings.Contains(diffOut, "+v2") { + t.Fatalf("expected diff output with v1/v2 changes, got: %q", diffOut) + } +} + +func initTestRepo(t *testing.T) string { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git is not installed") + } + + repo := t.TempDir() + runGit(t, repo, "init") + runGit(t, repo, "config", "user.name", "test-user") + runGit(t, repo, "config", "user.email", "test@example.com") + + readme := filepath.Join(repo, "README.md") + if err := os.WriteFile(readme, []byte("hello\n"), 0o644); err != nil { + t.Fatalf("write readme failed: %v", err) + } + runGit(t, repo, "add", "README.md") + runGit(t, repo, "commit", "-m", "initial commit") + + return repo +} + +func runGit(t *testing.T, repo string, args ...string) { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Dir = repo + cmd.Env = append(cmd.Environ(), + "GIT_TERMINAL_PROMPT=0", + "GIT_PAGER=cat", + "GIT_EDITOR=true", + ) + if runtime.GOOS == "windows" { + // Keep default behavior; this branch documents cross-platform intent. + } + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s failed: %v\n%s", strings.Join(args, " "), err, string(out)) + } +} diff --git a/tools/shell/shell.go b/tools/shell/shell.go index cabc63f..3ad2165 100644 --- a/tools/shell/shell.go +++ b/tools/shell/shell.go @@ -94,6 +94,9 @@ func (t *Tool) Call(ctx context.Context, input string) (string, error) { if runtime.GOOS == "windows" { // Windows 下使用 cmd /C 执行，兼容 date、dir 等内建命令。 cmd = exec.CommandContext(runCtx, "cmd", "/C", trimmed) + } else if requiresShellParsing(trimmed) { + // 包含管道、重定向等语法时，必须交给 shell 解释。 + cmd = exec.CommandContext(runCtx, "sh", "-c", trimmed) } else { cmd = exec.CommandContext(runCtx, base, parts[1:]...) } @@ -126,3 +129,7 @@ func normalizeWindowsCommand(command string) string { return command } } + +func requiresShellParsing(command string) bool { + return strings.ContainsAny(command, "|&;<>()$`\\\n") +} diff --git a/tools/shell/shell_test.go b/tools/shell/shell_test.go index 72afaeb..8731516 100644 --- a/tools/shell/shell_test.go +++ b/tools/shell/shell_test.go @@ -40,3 +40,27 @@ func TestCallWindowsDateIsNonInteractive(t *testing.T) { t.Fatal("expected non-empty output for date command") } } + +func TestRequiresShellParsing(t *testing.T) { + if !requiresShellParsing("echo hi | cat") { + t.Fatal("expected pipe command to require shell parsing") + } + if requiresShellParsing("echo hello") { + t.Fatal("expected simple command to not require shell parsing") + } +} + +func TestCallSupportsPipeOnUnix(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("unix-only behavior test") + } + tool := New([]string{"echo"}, ".", 3*time.Second, 4000, nil) + out, err := tool.Call(context.Background(), "printf hello | wc -c") + if err != nil { + t.Fatalf("expected pipeline command success, got err=%v output=%q", err, out) + } + trimmed := strings.TrimSpace(out) + if trimmed != "5" { + t.Fatalf("expected output 5, got %q", trimmed) + } +}