diff --git a/.editorconfig b/.editorconfig index a37a0f9b..c0719870 100644 --- a/.editorconfig +++ b/.editorconfig @@ -16,8 +16,9 @@ trim_trailing_whitespace = true indent_style = space indent_size = 2 -# 4 space indentation -[*.md] +# Matches multiple files with brace expansion notation +# Set default charset +[*.{md}] indent_style = space indent_size = 4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d0177182..60ef72ae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,6 +64,12 @@ jobs: - name: check run: task check + - name: Publish Test Results + if: always() + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + files: "dist/md-lint-issues.xml" + - name: build run: task clean build diff --git a/.markdownlint.jsonc b/.markdownlint.jsonc index 940758ca..525939c8 100644 --- a/.markdownlint.jsonc +++ b/.markdownlint.jsonc @@ -1,34 +1,35 @@ { - // Example markdownlint JSON(C) configuration with all properties set to their default value - - // Default state for all rules "default": true, - - // not enforcing line break a specific lenght + "MD012": false, "MD013": false, - "MD007": { "indent": 4 }, - - // allow Multiple headings with the same content "MD024": false, - - // to be consistent with prettier + "MD025": false, "MD030": { - "ul_single": 3, - "ul_multi": 3 + "ul_single": 1, + "ul_multi": 1 }, - - // allow some html "MD033": { - "allowed_elements": ["figure", "figcaption", "div"] + "allowed_elements": [ + "a", + "br", + "cite", + "details", + "div", + "figcaption", + "figure", + "img", + "p", + "span", + "summary" + ] }, - - // allow missing img alt text + "MD035": false, + "MD041": false, "MD045": false, - - // disabled due to admonition / tabs using indented blocks - "MD046": false - //"MD046": { "style": "fenced" } + "MD046": false, + "MD059": false, + "MD060": false } diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..1ec91637 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,11 @@ + +--- +repos: + - repo: local + hooks: + + - id: markdownlint + name: check:rumdl + entry: task check:rumdl + language: python + pass_filenames: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 466ca17d..1704a607 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,9 +31,9 @@ We suggest to use a specialized markdown editor such as [obsidian](https://obsid The following tools you need locally to get started: -- [poetry](https://python-poetry.org/) -- [task](https://taskfile.dev/) -- git, markdown editor +- [poetry](https://python-poetry.org/) +- [task](https://taskfile.dev/) +- git, markdown editor On a few OS distributions (e.g. Arch Linux) the tool/binary is named `go-task`. @@ -71,26 +71,26 @@ Have a look at the [mkdocs-material documentation](https://squidfunk.github.io/m
Extend section -- always create a directory + `index.md`, e.g. `my-topic/index.md` ([Example](https://github.com/eccenca/documentation.eccenca.com/tree/main/docs/automate/cmemc-command-line-interface)) -- add new pages to the `.pages` file to add them in the right order and with correct title to the menu ([Example](https://github.com/eccenca/documentation.eccenca.com/blob/main/docs/automate/cmemc-command-line-interface/.pages)) -- put images side by side to the `index.md` ([Example](https://github.com/eccenca/documentation.eccenca.com/tree/main/docs/release-notes/corporate-memory-22-1)) -- do not use images for icons esp. icons from the application - - use eccenca icons, e.g. [:eccenca-application-queries:](https://github.com/eccenca/documentation.eccenca.com/blob/main/overrides/.icons/eccenca/application-queries.svg) -> [list](https://github.com/eccenca/documentation.eccenca.com/tree/main/overrides/.icons/eccenca) - - use theme icons where no eccenca icon is available -> [list](https://squidfunk.github.io/mkdocs-material/reference/icons-emojis/#search) -- name image files properly (not just `Screenshot.xxx.png`, [Example](https://github.com/eccenca/documentation.eccenca.com/tree/main/docs/release-notes/corporate-memory-22-1)) -- used advanced features where suitable - - [Admonitions](https://squidfunk.github.io/mkdocs-material/reference/admonitions/#usage) (esp. use notes and warnings where needed) -> see Admonition section for more details - - [Code Blocks](https://squidfunk.github.io/mkdocs-material/reference/code-blocks/#usage) (e.g. enable highlightning and add a title) - - [Content Tabs](https://squidfunk.github.io/mkdocs-material/reference/content-tabs/#usage) (to structure complex pages) -- code blocks: - - do not use line numbers except you refer to it in the text - - use correct syntax highlightning (often used: `shell-session`, `bash`, `sparql`, `turtle`, `json`) -> [list of syntax IDs](https://pygments.org/docs/lexers/) - - do not confuse `shell-session` with `bash` (the first is a terminal output, the latter a script syntax) - - when using `shell-session`, use `$` as the prompt -- Links: - - do not use absolute links for internal documents, e.g. `https://documentation.eccenca.com/latest/...` - - do not use base-relative links, e.g. `/automate/...` - - use relative links to `index.md` files +- always create a directory + `index.md`, e.g. `my-topic/index.md` ([Example](https://github.com/eccenca/documentation.eccenca.com/tree/main/docs/automate/cmemc-command-line-interface)) +- add new pages to the `.pages` file to add them in the right order and with correct title to the menu ([Example](https://github.com/eccenca/documentation.eccenca.com/blob/main/docs/automate/cmemc-command-line-interface/.pages)) +- put images side by side to the `index.md` ([Example](https://github.com/eccenca/documentation.eccenca.com/tree/main/docs/release-notes/corporate-memory-22-1)) +- do not use images for icons esp. icons from the application + - use eccenca icons, e.g. [:eccenca-application-queries:](https://github.com/eccenca/documentation.eccenca.com/blob/main/overrides/.icons/eccenca/application-queries.svg) -> [list](https://github.com/eccenca/documentation.eccenca.com/tree/main/overrides/.icons/eccenca) + - use theme icons where no eccenca icon is available -> [list](https://squidfunk.github.io/mkdocs-material/reference/icons-emojis/#search) +- name image files properly (not just `Screenshot.xxx.png`, [Example](https://github.com/eccenca/documentation.eccenca.com/tree/main/docs/release-notes/corporate-memory-22-1)) +- used advanced features where suitable + - [Admonitions](https://squidfunk.github.io/mkdocs-material/reference/admonitions/#usage) (esp. use notes and warnings where needed) -> see Admonition section for more details + - [Code Blocks](https://squidfunk.github.io/mkdocs-material/reference/code-blocks/#usage) (e.g. enable highlightning and add a title) + - [Content Tabs](https://squidfunk.github.io/mkdocs-material/reference/content-tabs/#usage) (to structure complex pages) +- code blocks: + - do not use line numbers except you refer to it in the text + - use correct syntax highlightning (often used: `shell-session`, `bash`, `sparql`, `turtle`, `json`) -> [list of syntax IDs](https://pygments.org/docs/lexers/) + - do not confuse `shell-session` with `bash` (the first is a terminal output, the latter a script syntax) + - when using `shell-session`, use `$` as the prompt +- Links: + - do not use absolute links for internal documents, e.g. `https://documentation.eccenca.com/latest/...` + - do not use base-relative links, e.g. `/automate/...` + - use relative links to `index.md` files
@@ -104,13 +104,13 @@ On this page is search function for icons available as well.
Extend section -|Admonition Name|Used For |Example| -|---------------|---------|-------| -|Info |Information in documentation provides details about a topic or process that the reader needs to know. It is usually essential and relevant to the main subject of the document.|Statement annotations provide a way to express knowledge about statements. This group is dedicated to properties that configure the Statement Annotation feature| -|Note |A note provides additional details that may or may not be directly related to the main topic. It could be an explanation, clarification, or an aside that the writer thinks would be helpful for the reader to know.|The graph selection drop-down might or might not be visible depending the existence of an (optional) EasyNav Module configuration. In case no specific module configuration exists or non has not has been set for the current workspace the graph selection will be shown. A EasyNav Module configuration pre-configures a graph. Thus, the dropdown will not be shown if such has been configured for the current workspace.| -|Abstract|An abstract is a brief summary that provides an overview of the main points or contents of a document. It typically appears at the beginning of a document and is intended to give the reader an idea of what to expect from the document.|This tutorial explores the benefits of using cloud computing in enterprise organizations. It discusses the advantages of cloud computing over traditional on-premises infrastructure, and provides guidance for migrating to the cloud.| -|Warning|It is used to convey the seriousness of the risk and the importance of taking necessary precautions to avoid harm.|If the remote file resource is used in more than one dataset, the other datasets are also affected by this command.| -|Tip|A tip is a type of admonition in documentation that provides a helpful suggestion or best practice related to the content of the document. It is typically used to guide the reader towards a more efficient or effective way of using a product or service, or to provide additional insights or recommendations.|We have the suggestion option as well; click on the +icon and select the suggestion mapping.| +| Admonition Name | Used For | Example | +| ---- ----------- | -------- | ------- | +| Info | Information in documentation provides details about a topic or process that the reader needs to know. It is usually essential and relevant to the main subject of the document. | Statement annotations provide a way to express knowledge about statements. This group is dedicated to properties that configure the Statement Annotation feature | +| Note | A note provides additional details that may or may not be directly related to the main topic. It could be an explanation, clarification, or an aside that the writer thinks would be helpful for the reader to know. | The graph selection drop-down might or might not be visible depending the existence of an (optional) EasyNav Module configuration. In case no specific module configuration exists or non has not has been set for the current workspace the graph selection will be shown. A EasyNav Module configuration pre-configures a graph. Thus, the dropdown will not be shown if such has been configured for the current workspace. | +| Abstract | An abstract is a brief summary that provides an overview of the main points or contents of a document. It typically appears at the beginning of a document and is intended to give the reader an idea of what to expect from the document. | This tutorial explores the benefits of using cloud computing in enterprise organizations. It discusses the advantages of cloud computing over traditional on-premises infrastructure, and provides guidance for migrating to the cloud. | +| Warning | It is used to convey the seriousness of the risk and the importance of taking necessary precautions to avoid harm. | If the remote file resource is used in more than one dataset, the other datasets are also affected by this command. | +| Tip | A tip is a type of admonition in documentation that provides a helpful suggestion or best practice related to the content of the document. It is typically used to guide the reader towards a more efficient or effective way of using a product or service, or to provide additional insights or recommendations. | We have the suggestion option as well; click on the +icon and select the suggestion mapping. | |Success|Success admonitions are a type of documentation element used to highlight successful outcomes or positive results associated with a particular task, process, or feature|Graph is created successfully.| |Bug|A bug admonition include a description of the bug or issue, as well as steps that the user can take to avoid or work around the problem. It may also include information about when the bug will be fixed or patched, if applicable.|Users may experience issues with the file saving feature when running this software on Windows 10. To avoid data loss or corruption, be sure to save your work frequently and consider using an external backup device. Our development team is working to resolve this issue in the next software update.| |Example|The example admonition is typically used in instructional or educational documents to clarify complex concepts or demonstrate how to perform a specific task.|To create a new email account, click on the "Sign Up" button on the homepage and enter your name, email address, and desired password. Be sure to choose a strong password with a mix of uppercase and lowercase letters, numbers, and special characters. Once you have entered your information, click "Create Account" to complete the process.| @@ -123,14 +123,14 @@ On this page is search function for icons available as well.
Extend section -- do not use a cluttered desktop -- do not show other esp. personal project artifacts than relevant for the tutorial / feature to show -- select cropping area carefully (omit backgrounds, lines on the edges, etc.) -- use the same or a similar area for similar screens -- all relevant elements should be clearly visible and not be truncated -- irrelevant elements / details should be omitted completely and not be half visible -- crop scrollbars (they can make edges look unclean, especially if a scrollbar is directly on an edge) -- keep an equal distance of all visible elements to the edges of the screenshot +- do not use a cluttered desktop +- do not show other esp. personal project artifacts then relevant for the tutorial / feature to show +- select cropping area carefully (omit backgrounds, lines on the edges, etc.) +- use the same or a similar area for similar screens +- all relevant elements should be clearly visible and not be truncated +- irrelevant elements / details should be omitted completely and not be half visible +- crop scrollbars (they can make edges look unclean, especially if a scrollbar is directly on an edge) +- keep an equal distance of all visible elements to the edges of the screenshot
@@ -138,12 +138,12 @@ On this page is search function for icons available as well.
The following files are generated from other sources and should not be changed: - -- cmemc Command Reference: [docs/automate/cmemc-command-line-interface/command-reference](https://github.com/eccenca/documentation.eccenca.com/tree/main/docs/automate/cmemc-command-line-interface/command-reference) + +- cmemc Command Reference: [docs/automate/cmemc-command-line-interface/command-reference](https://github.com/eccenca/documentation.eccenca.com/tree/main/docs/automate/cmemc-command-line-interface/command-reference) - to update, start `task update:cmemc` -- Coporate Memory Application Icons: [overrides/.icons/eccenca](https://github.com/eccenca/documentation.eccenca.com/tree/main/overrides/.icons/eccenca) +- Coporate Memory Application Icons: [overrides/.icons/eccenca](https://github.com/eccenca/documentation.eccenca.com/tree/main/overrides/.icons/eccenca) - to update, start `task update:icons` -- Shapes and Datatypes: [docs/explore-and-author/graph-exploration/building-a-customized-user-interface/node-shapes|property-shapes|datatype-reference](https://github.com/eccenca/documentation.eccenca.com/tree/main/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/node-shapes) +- Shapes and Datatypes: [docs/explore-and-author/graph-exploration/building-a-customized-user-interface/node-shapes|property-shapes|datatype-reference](https://github.com/eccenca/documentation.eccenca.com/tree/main/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/node-shapes) - to update, start `task update:shape-reference` Where applicable, we added the following disclaimer: `` diff --git a/README.md b/README.md index caa637d5..628d3e31 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,10 @@ Shared repository of the eccenca Corporate Memory documentation. -| Branch | Deployment | -| :--- | :--- | -| `main` | [https://dev.documentation.eccenca.com](https://dev.documentation.eccenca.com) | -| `published` | [https://documentation.eccenca.com](https://documentation.eccenca.com) | +| Branch | Deployment | +| :----- | :--------- | +| `main` | [https://dev.documentation.eccenca.com](https://dev.documentation.eccenca.com) | +| `published` | [https://documentation.eccenca.com](https://documentation.eccenca.com) | If you consider to contribute to this project, please have a look on [CONTRIBUTING.md](https://github.com/eccenca/documentation.eccenca.com/blob/main/CONTRIBUTING.md) @@ -27,4 +27,3 @@ This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 Inter [mkdocs-shield]: https://img.shields.io/badge/Made%20with-mkdocs-brightgreen [mkdocs]: https://www.mkdocs.org/ [markdown-shield]: https://img.shields.io/badge/Made%20with-Markdown-1f425f.svg - diff --git a/Taskfile.yml b/Taskfile.yml index 01d3ff32..6c81615e 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -51,6 +51,7 @@ tasks: desc: Run complete test suite deps: - check:links + - check:rumdl check:links: desc: Check outgoing links @@ -180,3 +181,36 @@ tasks: cmds: - poetry run mike serve -b {{.PUBLIC_BRANCH}} + format:fix: + desc: rumdl (md-lint) and apply possible style fixes + cmds: + - | + { + # recurse + find ./docs -type f -name '*.md' -print0 + + # no recursion (top-level only) + find . -maxdepth 1 -type f -name '*.md' -print0 + } | xargs -0 poetry run rumdl --config .markdownlint.jsonc check --fix + + check:rumdl: + desc: run rumdl (md-linter) for style issues + cmds: + - mkdir -p ./dist + - rm -f ./dist/md-lint-issues.xml + - | + { + # recurse + find ./docs -type f -name '*.md' -print0 + + # no recursion (top-level only) + find . -maxdepth 1 -type f -name '*.md' -print0 + } | xargs -0 poetry run rumdl --config .markdownlint.jsonc check --output-format junit > ./dist/md-lint-issues.xml || true + - | + { + # recurse + find ./docs -type f -name '*.md' -print0 + + # no recursion (top-level only) + find . -maxdepth 1 -type f -name '*.md' -print0 + } | xargs -0 poetry run rumdl --config .markdownlint.jsonc check diff --git a/data/plugins.json b/data/plugins.json index b206fc61..aaf8ea20 100644 --- a/data/plugins.json +++ b/data/plugins.json @@ -6629,6 +6629,121 @@ ], "pluginType": "customtask" }, + "cmem_plugin_semspect-task-Update": { + "pluginId": "cmem_plugin_semspect-task-Update", + "title": "Update SemSpect", + "categories": [ + "Uncategorized" + ], + "main_category": "Uncategorized", + "description": "Tell SemSpect to prepare a Knowledge Graph for visualization.", + "markdownDocumentation": null, + "pluginIcon": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgdmVyc2lvbj0iMS4xIgogICB2aWV3Qm94PSIzOTYgNDAyIDc2IDY1IgogICB3aWR0aD0iNzZwdCIKICAgaGVpZ2h0PSI2NXB0IgogICBpZD0ic3ZnODY0IgogICBzb2RpcG9kaTpkb2NuYW1lPSJsb2dvLnN2ZyIKICAgaW5rc2NhcGU6dmVyc2lvbj0iMS4xLjIgKGI4ZTI1YmU4LCAyMDIyLTAyLTA1KSIKICAgeG1sbnM6aW5rc2NhcGU9Imh0dHA6Ly93d3cuaW5rc2NhcGUub3JnL25hbWVzcGFjZXMvaW5rc2NhcGUiCiAgIHhtbG5zOnNvZGlwb2RpPSJodHRwOi8vc29kaXBvZGkuc291cmNlZm9yZ2UubmV0L0RURC9zb2RpcG9kaS0wLmR0ZCIKICAgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIgogICB4bWxuczpzdmc9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIgogICB4bWxuczpkYz0iaHR0cDovL3B1cmwub3JnL2RjL2VsZW1lbnRzLzEuMS8iPgogIDxzb2RpcG9kaTpuYW1lZHZpZXcKICAgICBpZD0ibmFtZWR2aWV3ODY2IgogICAgIHBhZ2Vjb2xvcj0iI2ZmZmZmZiIKICAgICBib3JkZXJjb2xvcj0iIzY2NjY2NiIKICAgICBib3JkZXJvcGFjaXR5PSIxLjAiCiAgICAgaW5rc2NhcGU6cGFnZXNoYWRvdz0iMiIKICAgICBpbmtzY2FwZTpwYWdlb3BhY2l0eT0iMC4wIgogICAgIGlua3NjYXBlOnBhZ2VjaGVja2VyYm9hcmQ9IjAiCiAgICAgaW5rc2NhcGU6ZG9jdW1lbnQtdW5pdHM9InB0IgogICAgIHNob3dncmlkPSJmYWxzZSIKICAgICBpbmtzY2FwZTp6b29tPSIxMi41NjU0ODUiCiAgICAgaW5rc2NhcGU6Y3g9IjYyLjE5NDE3NiIKICAgICBpbmtzY2FwZTpjeT0iNDMuNDEyNTciCiAgICAgaW5rc2NhcGU6d2luZG93LXdpZHRoPSIxOTIwIgogICAgIGlua3NjYXBlOndpbmRvdy1oZWlnaHQ9IjEwMjciCiAgICAgaW5rc2NhcGU6d2luZG93LXg9IjE3MjgiCiAgICAgaW5rc2NhcGU6d2luZG93LXk9IjI1IgogICAgIGlua3NjYXBlOndpbmRvdy1tYXhpbWl6ZWQ9IjAiCiAgICAgaW5rc2NhcGU6Y3VycmVudC1sYXllcj0ic3ZnODY0IgogICAgIHNob3dndWlkZXM9InRydWUiCiAgICAgaW5rc2NhcGU6Z3VpZGUtYmJveD0idHJ1ZSI+CiAgICA8c29kaXBvZGk6Z3VpZGUKICAgICAgIHBvc2l0aW9uPSI2MC41ODA3NjYsMzcuODM4NzcyIgogICAgICAgb3JpZW50YXRpb249IjAsLTEiCiAgICAgICBpZD0iZ3VpZGUxNDQ4MiIgLz4KICAgIDxzb2RpcG9kaTpndWlkZQogICAgICAgcG9zaXRpb249IjU5Ljg0MzA2NCwzOC4zNDU0MzYiCiAgICAgICBvcmllbnRhdGlvbj0iMSwwIgogICAgICAgaWQ9Imd1aWRlMTQ0ODQiIC8+CiAgICA8c29kaXBvZGk6Z3VpZGUKICAgICAgIHBvc2l0aW9uPSI2Ny45OTgzMjMsMzMuODc0NjM1IgogICAgICAgb3JpZW50YXRpb249IjEsMCIKICAgICAgIGlkPSJndWlkZTE0NDg2IiAvPgogIDwvc29kaXBvZGk6bmFtZWR2aWV3PgogIDxtZXRhZGF0YQogICAgIGlkPSJtZXRhZGF0YTgyNCI+IFByb2R1Y2VkIGJ5IE9tbmlHcmFmZmxlIDYuNi4yIDxkYzpkYXRlPjIwMjQtMDUtMDYgMDk6NDM6NTYgKzAwMDA8L2RjOmRhdGU+CjwvbWV0YWRhdGE+CiAgPGRlZnMKICAgICBpZD0iZGVmczgyOCI+CiAgICA8Zm9udC1mYWNlCiAgICAgICBmb250LWZhbWlseT0iTHVjaWRhIEdyYW5kZSIKICAgICAgIGZvbnQtc2l6ZT0iMTMiCiAgICAgICBwYW5vc2UtMT0iMiAxMSA2IDAgNCA1IDIgMiAyIDQiCiAgICAgICB1bml0cy1wZXItZW09IjEwMDAiCiAgICAgICB1bmRlcmxpbmUtcG9zaXRpb249Ii05Ny42NTYyNSIKICAgICAgIHVuZGVybGluZS10aGlja25lc3M9IjQ4LjgyODEyNSIKICAgICAgIHNsb3BlPSIwIgogICAgICAgeC1oZWlnaHQ9IjUzMC4yNzM0NCIKICAgICAgIGNhcC1oZWlnaHQ9IjcyMi42NTYyNSIKICAgICAgIGFzY2VudD0iOTY2Ljc5NjkiCiAgICAgICBkZXNjZW50PSItMjEwLjkzNzUiCiAgICAgICBmb250LXdlaWdodD0iNTAwIgogICAgICAgaWQ9ImZvbnQtZmFjZTgyNiI+CiAgICAgIDxmb250LWZhY2Utc3JjPgogICAgICAgIDxmb250LWZhY2UtbmFtZQogICAgICAgICAgIG5hbWU9Ikx1Y2lkYUdyYW5kZSIgLz4KICAgICAgPC9mb250LWZhY2Utc3JjPgogICAgPC9mb250LWZhY2U+CiAgPC9kZWZzPgogIDxlbGxpcHNlCiAgICAgY3g9IjQ1NS40MDE1NSIKICAgICBjeT0iNDM0LjEwNzI0IgogICAgIGZpbGw9IiNjMmQxZTMiCiAgICAgaWQ9ImNpcmNsZTgzNCIKICAgICBzdHlsZT0iZmlsbC1vcGFjaXR5OjE7c3Ryb2tlOm5vbmU7c3Ryb2tlLXdpZHRoOjAuODY4MjQxO3N0cm9rZS1kYXNoYXJyYXk6bm9uZTtzdHJva2Utb3BhY2l0eToxIgogICAgIHJ4PSIxMC42ODIzNzQiCiAgICAgcnk9IjEwLjg0MTkzOSIgLz4KICA8ZWxsaXBzZQogICAgIGN4PSI0MjkuMjMxMDUiCiAgICAgY3k9IjQzNC4xNTYxIgogICAgIGZpbGw9IiNjMmQxZTQiCiAgICAgaWQ9ImNpcmNsZTgzNiIKICAgICBzdHlsZT0iZmlsbC1vcGFjaXR5OjE7c3Ryb2tlOm5vbmU7c3Ryb2tlLXdpZHRoOjAuODY4MjQxO3N0cm9rZS1kYXNoYXJyYXk6bm9uZTtzdHJva2Utb3BhY2l0eToxIgogICAgIHJ4PSIyNy4yMzkwMjciCiAgICAgcnk9IjI3LjY0NTkwMyIgLz4KICA8ZWxsaXBzZQogICAgIGN4PSI0NTUuNDUwODQiCiAgICAgY3k9IjQzNC4wNTcxOSIKICAgICBmaWxsPSIjYzJkMWUzIgogICAgIGlkPSJjaXJjbGU4MzgiCiAgICAgc3R5bGU9ImZpbGwtb3BhY2l0eToxO3N0cm9rZTpub25lO3N0cm9rZS13aWR0aDowLjg2ODI0MTtzdHJva2UtZGFzaGFycmF5Om5vbmU7c3Ryb2tlLW9wYWNpdHk6MSIKICAgICByeD0iMTAuNjgyMzc0IgogICAgIHJ5PSIxMC44NDE5MzkiIC8+CiAgPGVsbGlwc2UKICAgICBjeD0iNDI5LjIzMTA1IgogICAgIGN5PSI0MzQuNTgwMTQiCiAgICAgZmlsbD0iI2MyZDFlNCIKICAgICBpZD0iY2lyY2xlODQwIgogICAgIHN0eWxlPSJmaWxsLW9wYWNpdHk6MTtzdHJva2U6bm9uZTtzdHJva2Utd2lkdGg6MC44NjgyNDE7c3Ryb2tlLWRhc2hhcnJheTpub25lO3N0cm9rZS1vcGFjaXR5OjEiCiAgICAgcng9IjI3LjIzOTAyNyIKICAgICByeT0iMjcuNjQ1OTAzIiAvPgogIDxlbGxpcHNlCiAgICAgY3g9IjQxNC4zNzM0MSIKICAgICBjeT0iNDM0LjE1NjA3IgogICAgIGZpbGw9IiM2YzcxNzkiCiAgICAgaWQ9ImNpcmNsZTg0MiIKICAgICBzdHlsZT0iZmlsbC1vcGFjaXR5OjE7c3Ryb2tlOm5vbmU7c3Ryb2tlLXdpZHRoOjAuODY4MjQxO3N0cm9rZS1kYXNoYXJyYXk6bm9uZTtzdHJva2Utb3BhY2l0eToxIgogICAgIHJ4PSI1Ljc3Nzk3NDYiCiAgICAgcnk9IjUuODY0MjgxNyIgLz4KICA8ZWxsaXBzZQogICAgIGN4PSI0MjkuMjMxMDUiCiAgICAgY3k9IjQzNC4xNTYwNyIKICAgICBmaWxsPSIjYWIyYzBiIgogICAgIGlkPSJjaXJjbGU4NDQiCiAgICAgc3R5bGU9ImZpbGwtb3BhY2l0eToxO3N0cm9rZTpub25lO3N0cm9rZS13aWR0aDowLjg2ODI0MTtzdHJva2UtZGFzaGFycmF5Om5vbmU7c3Ryb2tlLW9wYWNpdHk6MSIKICAgICByeD0iNS43Nzc5NzQ2IgogICAgIHJ5PSI1Ljg2NDI4MTciIC8+CiAgPGVsbGlwc2UKICAgICBjeD0iNDIxLjk2NzMyIgogICAgIGN5PSI0MjAuNzUyMDQiCiAgICAgZmlsbD0iIzZjNzE3OSIKICAgICBpZD0iY2lyY2xlODQ2IgogICAgIHN0eWxlPSJmaWxsLW9wYWNpdHk6MTtzdHJva2U6bm9uZTtzdHJva2Utd2lkdGg6MC44NjgyNDE7c3Ryb2tlLWRhc2hhcnJheTpub25lO3N0cm9rZS1vcGFjaXR5OjEiCiAgICAgcng9IjUuNzc3OTc0NiIKICAgICByeT0iNS44NjQyODE3IiAvPgogIDxlbGxpcHNlCiAgICAgY3g9IjQzNi44MjQ5MiIKICAgICBjeT0iNDIwLjc1MjA0IgogICAgIGZpbGw9IiM2YzcxNzkiCiAgICAgaWQ9ImNpcmNsZTg0OCIKICAgICBzdHlsZT0iZmlsbC1vcGFjaXR5OjE7c3Ryb2tlOm5vbmU7c3Ryb2tlLXdpZHRoOjAuODY4MjQxO3N0cm9rZS1kYXNoYXJyYXk6bm9uZTtzdHJva2Utb3BhY2l0eToxIgogICAgIHJ4PSI1Ljc3Nzk3NDYiCiAgICAgcnk9IjUuODY0MjgxNyIgLz4KICA8ZWxsaXBzZQogICAgIGN4PSI0MjEuOTY3MzIiCiAgICAgY3k9IjQ0Ny41NjAxNSIKICAgICBmaWxsPSIjNmM3MTc5IgogICAgIGlkPSJjaXJjbGU4NTAiCiAgICAgc3R5bGU9ImZpbGwtb3BhY2l0eToxO3N0cm9rZTpub25lO3N0cm9rZS13aWR0aDowLjg2ODI0MTtzdHJva2UtZGFzaGFycmF5Om5vbmU7c3Ryb2tlLW9wYWNpdHk6MSIKICAgICByeD0iNS43Nzc5NzQ2IgogICAgIHJ5PSI1Ljg2NDI4MTciIC8+CiAgPGVsbGlwc2UKICAgICBjeD0iNDQ0LjA4ODY1IgogICAgIGN5PSI0MzQuMTU2MDciCiAgICAgZmlsbD0iIzZjNzE3OSIKICAgICBpZD0iY2lyY2xlODUyIgogICAgIHN0eWxlPSJmaWxsLW9wYWNpdHk6MTtzdHJva2U6bm9uZTtzdHJva2Utd2lkdGg6MC44NjgyNDE7c3Ryb2tlLWRhc2hhcnJheTpub25lO3N0cm9rZS1vcGFjaXR5OjEiCiAgICAgcng9IjUuNzc3OTc0NiIKICAgICByeT0iNS44NjQyODE3IiAvPgogIDxlbGxpcHNlCiAgICAgY3g9IjQzNi44MjQ5MiIKICAgICBjeT0iNDQ3LjU2MDE1IgogICAgIGZpbGw9IiM2YzcxNzkiCiAgICAgaWQ9ImNpcmNsZTg1NCIKICAgICBzdHlsZT0iZmlsbC1vcGFjaXR5OjE7c3Ryb2tlOm5vbmU7c3Ryb2tlLXdpZHRoOjAuODY4MjQxO3N0cm9rZS1kYXNoYXJyYXk6bm9uZTtzdHJva2Utb3BhY2l0eToxIgogICAgIHJ4PSI1Ljc3Nzk3NDYiCiAgICAgcnk9IjUuODY0MjgxNyIgLz4KICA8cGF0aAogICAgIHNvZGlwb2RpOnR5cGU9InN0YXIiCiAgICAgc3R5bGU9ImZpbGw6IzZjNzE3OTtmaWxsLW9wYWNpdHk6MTtmaWxsLXJ1bGU6ZXZlbm9kZCIKICAgICBpZD0icGF0aDE0MjA0IgogICAgIGlua3NjYXBlOmZsYXRzaWRlZD0idHJ1ZSIKICAgICBzb2RpcG9kaTpzaWRlcz0iMyIKICAgICBzb2RpcG9kaTpjeD0iNzkuNTAzNDY0IgogICAgIHNvZGlwb2RpOmN5PSI3MC4wNzczMTYiCiAgICAgc29kaXBvZGk6cjE9IjcuNjc2Mzc2MyIKICAgICBzb2RpcG9kaTpyMj0iMy40NzEzODYyIgogICAgIHNvZGlwb2RpOmFyZzE9IjIuMDg5OTQyNSIKICAgICBzb2RpcG9kaTphcmcyPSIzLjEzNzE0IgogICAgIGlua3NjYXBlOnJvdW5kZWQ9IjAiCiAgICAgaW5rc2NhcGU6cmFuZG9taXplZD0iMCIKICAgICBkPSJtIDc1LjY5NDkxNCw3Ni43NDIyNzcgLTAuMDU5MiwtMTMuMjk1NzQyIDExLjU0NDA1MSw2LjU5NjYwMiB6IgogICAgIHRyYW5zZm9ybT0ibWF0cml4KDAuNzUsMCwwLDAuNzUsMzk5LjExNjI4LDM4MS41NzYzMykiCiAgICAgaW5rc2NhcGU6dHJhbnNmb3JtLWNlbnRlci14PSItMC4yMTk3MDEzNyIKICAgICBpbmtzY2FwZTp0cmFuc2Zvcm0tY2VudGVyLXk9Ii0xLjEwNjg1OTYiIC8+Cjwvc3ZnPgo=", + "properties": { + "base_url": { + "name": "base_url", + "title": "The URL of the SemSpect application.This needs to be accessible from 'within' DataIntegration.", + "description": "", + "type": "string", + "parameterType": "string", + "value": "http://semspect:8080/semspect/", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "database_id": { + "name": "database_id", + "title": "The SemSpect database ID. Not existing databases will be created.", + "description": "", + "type": "string", + "parameterType": "string", + "value": "cmem", + "advanced": false, + "visibleInDialog": true, + "properties": {} + }, + "graph": { + "name": "graph", + "title": "Knowledge Graph", + "description": "", + "type": "string", + "parameterType": "string", + "value": null, + "advanced": false, + "visibleInDialog": true, + "properties": {} + } + }, + "properties_advanced": { + "dataplatform_base": { + "name": "dataplatform_base", + "title": "The URL of the DataPlatform application.This needs to be accessible from 'within' SemSpect.", + "description": "", + "type": "string", + "parameterType": "string", + "value": "", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "timeout": { + "name": "timeout", + "title": "Timeout (in seconds) for the overall indexing activity.", + "description": "", + "type": "string", + "parameterType": "Long", + "value": "300", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "request_timeout": { + "name": "request_timeout", + "title": "Timeout (in seconds) for individual Semspect requests", + "description": "", + "type": "string", + "parameterType": "Long", + "value": "10", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "ignore_proxy": { + "name": "ignore_proxy", + "title": "ignore_proxy: Ignore system settings for HTTP proxies for the requests to semspect.", + "description": "", + "type": "string", + "parameterType": "boolean", + "value": "false", + "advanced": true, + "visibleInDialog": true, + "properties": {} + }, + "verify_ssl": { + "name": "verify_ssl", + "title": "verify_ssl: If disabled, the plugin will accept any TLS certificate presented by the server and will ignore hostname mismatches and/or expired certificates, which will make the requests vulnerable to man-in-the-middle (MitM) attacks. (use for testing only)", + "description": "", + "type": "string", + "parameterType": "boolean", + "value": "true", + "advanced": true, + "visibleInDialog": true, + "properties": {} + } + }, + "actions": {}, + "required": [ + "graph" + ], + "distanceMeasureRange": null, + "backendType": "python", + "is_deprecated": false, + "tags": [ + "WorkflowTask", + "PythonPlugin" + ], + "pluginType": "customtask" + }, "eccencaDataPlatformGraphStoreFileUploadOperator": { "pluginId": "eccencaDataPlatformGraphStoreFileUploadOperator", "title": "Upload File to Knowledge Graph", diff --git a/docs/automate/cmemc-command-line-interface/command-reference/admin/acl/index.md b/docs/automate/cmemc-command-line-interface/command-reference/admin/acl/index.md index 3b9eec83..7578d65a 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/admin/acl/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/admin/acl/index.md @@ -6,7 +6,9 @@ tags: - Security - cmemc --- + # admin acl Command Group + List, create, delete and modify and review access conditions. @@ -19,7 +21,7 @@ With this command group, you can manage and inspect access conditions in eccenca List access conditions. ```shell-session title="Usage" -$ cmemc admin acl list [OPTIONS] +cmemc admin acl list [OPTIONS] ``` @@ -45,7 +47,7 @@ This command retrieves and lists all access conditions, which are manageable by Inspect an access condition. ```shell-session title="Usage" -$ cmemc admin acl inspect [OPTIONS] ACCESS_CONDITION_ID +cmemc admin acl inspect [OPTIONS] ACCESS_CONDITION_ID ``` @@ -68,7 +70,7 @@ $ cmemc admin acl inspect [OPTIONS] ACCESS_CONDITION_ID Create an access condition. ```shell-session title="Usage" -$ cmemc admin acl create [OPTIONS] +cmemc admin acl create [OPTIONS] ``` @@ -87,7 +89,7 @@ A special case are dynamic access conditions, based on a SPARQL query: Here you ```shell-session title="Example" -$ cmemc admin acl create --group local-users --write-graph https://example.org/ +cmemc admin acl create --group local-users --write-graph https://example.org/ ``` @@ -135,7 +137,7 @@ $ cmemc admin acl create --group local-users --write-graph https://example.org/ Update an access condition. ```shell-session title="Usage" -$ cmemc admin acl update [OPTIONS] ACCESS_CONDITION_ID +cmemc admin acl update [OPTIONS] ACCESS_CONDITION_ID ``` @@ -183,7 +185,7 @@ Given an access condition URL, you can change specific options to new values. Delete access conditions. ```shell-session title="Usage" -$ cmemc admin acl delete [OPTIONS] [ACCESS_CONDITION_IDS]... +cmemc admin acl delete [OPTIONS] [ACCESS_CONDITION_IDS]... ``` @@ -279,7 +281,7 @@ $ cmemc admin acl export :my-acl-iri Import access conditions from a JSON file. ```shell-session title="Usage" -$ cmemc admin acl import [OPTIONS] INPUT_FILE +cmemc admin acl import [OPTIONS] INPUT_FILE ``` @@ -290,12 +292,12 @@ This command imports access conditions from a JSON file that was created using t If `--replace` is specified, existing access conditions with matching IRIs will be deleted before importing. Otherwise, the import will skip if an access condition with the same IRI already exists. ```shell-session title="Example" -$ cmemc admin acl import acls.json +cmemc admin acl import acls.json ``` ```shell-session title="Example" -$ cmemc admin acl import --replace acls.json +cmemc admin acl import --replace acls.json ``` @@ -313,7 +315,7 @@ $ cmemc admin acl import --replace acls.json Review grants for a given account. ```shell-session title="Usage" -$ cmemc admin acl review [OPTIONS] USER +cmemc admin acl review [OPTIONS] USER ``` @@ -331,4 +333,3 @@ The output of the command is a list of grants the account has based on your inpu --raw Outputs raw JSON. --group TEXT Add groups to the review request (what-if-scenario). ``` - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/admin/client/index.md b/docs/automate/cmemc-command-line-interface/command-reference/admin/client/index.md index 7e8b3e67..15d4a72f 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/admin/client/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/admin/client/index.md @@ -7,7 +7,9 @@ tags: - Security - cmemc --- + # admin client Command Group + List client accounts, get or generate client account secrets. @@ -24,7 +26,7 @@ In case your Corporate Memory deployment does not use the default deployment lay List client accounts. ```shell-session title="Usage" -$ cmemc admin client list [OPTIONS] +cmemc admin client list [OPTIONS] ``` @@ -51,7 +53,7 @@ Outputs a list of client accounts, which can be used to get an overview as well Get or generate a new secret for a client account. ```shell-session title="Usage" -$ cmemc admin client secret [OPTIONS] CLIENT_ID +cmemc admin client secret [OPTIONS] CLIENT_ID ``` @@ -73,7 +75,7 @@ This command retrieves or generates a new secret for a client account from a rea Open clients in the browser. ```shell-session title="Usage" -$ cmemc admin client open [CLIENT_IDS]... +cmemc admin client open [CLIENT_IDS]... ``` @@ -84,4 +86,3 @@ With this command, you can open a client in the keycloak web interface in your b The command accepts multiple client IDs which results in opening multiple browser tabs. - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/admin/index.md b/docs/automate/cmemc-command-line-interface/command-reference/admin/index.md index 2fe523f7..8308369c 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/admin/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/admin/index.md @@ -5,7 +5,9 @@ icon: material/key-link tags: - cmemc --- + # admin Command Group + Import bootstrap data, backup/restore workspace or get status. @@ -18,7 +20,7 @@ This command group consists of commands for setting up and configuring eccenca C Output health and version information. ```shell-session title="Usage" -$ cmemc admin status [OPTIONS] +cmemc admin status [OPTIONS] ``` @@ -31,7 +33,7 @@ Additionally, this command informs you in one of these cases: (1) A warning, if To get status information of all configured deployments use this command in combination with parallel. ```shell-session title="Example" -$ cmemc config list | parallel --ctag cmemc -c {} admin status +cmemc config list | parallel --ctag cmemc -c {} admin status ``` @@ -66,7 +68,7 @@ $ cmemc config list | parallel --ctag cmemc -c {} admin status Fetch and output an access token. ```shell-session title="Usage" -$ cmemc admin token [OPTIONS] +cmemc admin token [OPTIONS] ``` @@ -75,7 +77,7 @@ $ cmemc admin token [OPTIONS] This command can be used to check for correct authentication as well as to use the token with wget / curl or similar standard tools: ```shell-session title="Example" -$ curl -H "Authorization: Bearer $(cmemc -c my admin token)" $(cmemc -c my config get DP_API_ENDPOINT)/api/custom/slug +curl -H "Authorization: Bearer $(cmemc -c my admin token)" $(cmemc -c my config get DP_API_ENDPOINT)/api/custom/slug ``` @@ -94,4 +96,3 @@ Please be aware that this command can reveal secrets which you might not want to --ttl Output information about the lifetime of the access token. In combination with --raw, it outputs the TTL in seconds. ``` - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/admin/metrics/index.md b/docs/automate/cmemc-command-line-interface/command-reference/admin/metrics/index.md index 569f808f..d29d5084 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/admin/metrics/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/admin/metrics/index.md @@ -5,7 +5,9 @@ icon: material/chart-line-variant tags: - cmemc --- + # admin metrics Command Group + List and get metrics. @@ -14,7 +16,7 @@ This command group consists of commands for reading and listing internal monitor Each metric family can consist of different samples identified by labels with a name and a value (dimensions). A metric has a specific type (counter, gauge, summary and histogram) and additional metadata. -Please have a look at https://prometheus.io/docs/concepts/data_model/ for further details. +Please have a look at for further details. ## admin metrics get @@ -22,7 +24,7 @@ Please have a look at https://prometheus.io/docs/concepts/data_model/ for furthe Get sample data of a metric. ```shell-session title="Usage" -$ cmemc admin metrics get [OPTIONS] METRIC_ID +cmemc admin metrics get [OPTIONS] METRIC_ID ``` @@ -56,7 +58,7 @@ A metric of a specific job is identified by a metric ID. Possible metric IDs of Inspect a metric. ```shell-session title="Usage" -$ cmemc admin metrics inspect [OPTIONS] METRIC_ID +cmemc admin metrics inspect [OPTIONS] METRIC_ID ``` @@ -77,7 +79,7 @@ This command outputs the data of a metric. The first table includes basic metada List metrics for a specific job. ```shell-session title="Usage" -$ cmemc admin metrics list [OPTIONS] +cmemc admin metrics list [OPTIONS] ``` @@ -97,4 +99,3 @@ For each metric, the output table shows the metric ID, the type of the metric, a --raw Outputs (sorted) JSON dict, parsed from the metrics API output. ``` - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/admin/migration/index.md b/docs/automate/cmemc-command-line-interface/command-reference/admin/migration/index.md index 0ef7ba6b..57ca9090 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/admin/migration/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/admin/migration/index.md @@ -5,7 +5,9 @@ icon: material/database-arrow-up-outline tags: - cmemc --- + # admin migration Command Group + List and apply migration recipes. @@ -22,7 +24,7 @@ The following tags are important: `system` recipes target data structures which List migration recipies. ```shell-session title="Usage" -$ cmemc admin migration list [OPTIONS] +cmemc admin migration list [OPTIONS] ``` @@ -48,7 +50,7 @@ This command lists all available migration recipies Execute needed migration recipes. ```shell-session title="Usage" -$ cmemc admin migration execute [OPTIONS] [MIGRATION_ID] +cmemc admin migration execute [OPTIONS] [MIGRATION_ID] ``` @@ -72,4 +74,3 @@ Here are some argument examples, in order to see how to use this command: execut --test-only Only test, do not execute migrations. --id-only Lists only recipe identifier. ``` - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/admin/store/index.md b/docs/automate/cmemc-command-line-interface/command-reference/admin/store/index.md index 98baa329..dcf38b14 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/admin/store/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/admin/store/index.md @@ -6,7 +6,9 @@ tags: - SPARQL - cmemc --- + # admin store Command Group + Import, export and bootstrap the knowledge graph store. @@ -19,7 +21,7 @@ This command group consist of commands to administrate the knowledge graph store Create showcase data. ```shell-session title="Usage" -$ cmemc admin store showcase [OPTIONS] +cmemc admin store showcase [OPTIONS] ``` @@ -49,7 +51,7 @@ This command creates a showcase scenario of multiple graphs including integratio Update/Import or remove bootstrap data. ```shell-session title="Usage" -$ cmemc admin store bootstrap [OPTIONS] +cmemc admin store bootstrap [OPTIONS] ``` @@ -106,7 +108,7 @@ This command will create lots of load on the server. It can take a long time to Restore graphs from a ZIP archive. ```shell-session title="Usage" -$ cmemc admin store import BACKUP_FILE +cmemc admin store import BACKUP_FILE ``` @@ -119,4 +121,3 @@ The command will load a single backup ZIP archive into the triple store by repla This command will create lots of load on the server. It can take a long time to complete. The backup file will be transferred to the server, then unzipped and imported graph by graph. After the initial transfer the network connection is not used anymore and may be closed by proxies. This does not mean that the import failed. - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/admin/user/index.md b/docs/automate/cmemc-command-line-interface/command-reference/admin/user/index.md index e0482e93..5ee3e668 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/admin/user/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/admin/user/index.md @@ -7,7 +7,9 @@ tags: - Security - cmemc --- + # admin user Command Group + List, create, delete and modify user accounts. @@ -24,7 +26,7 @@ In case your Corporate Memory deployment does not use the default deployment lay List user accounts. ```shell-session title="Usage" -$ cmemc admin user list [OPTIONS] +cmemc admin user list [OPTIONS] ``` @@ -50,7 +52,7 @@ Outputs a list of user accounts, which can be used to get an overview as well as Create a user account. ```shell-session title="Usage" -$ cmemc admin user create USERNAME +cmemc admin user create USERNAME ``` @@ -69,7 +71,7 @@ This command creates a new user account. Update a user account. ```shell-session title="Usage" -$ cmemc admin user update [OPTIONS] USERNAME +cmemc admin user update [OPTIONS] USERNAME ``` @@ -100,7 +102,7 @@ For each data value, a separate option needs to be used. All options can be comb Delete user accounts. ```shell-session title="Usage" -$ cmemc admin user delete [OPTIONS] [USERNAMES]... +cmemc admin user delete [OPTIONS] [USERNAMES]... ``` @@ -133,7 +135,7 @@ This command deletes user accounts from a realm. Change the password of a user account. ```shell-session title="Usage" -$ cmemc admin user password [OPTIONS] USERNAME +cmemc admin user password [OPTIONS] USERNAME ``` @@ -163,7 +165,7 @@ With this command, the password of a user account can be changed. The default ex Open user in the browser. ```shell-session title="Usage" -$ cmemc admin user open [USERNAMES]... +cmemc admin user open [USERNAMES]... ``` @@ -174,4 +176,3 @@ With this command, you can open a user in the keycloak console in your browser t The command accepts multiple usernames which results in opening multiple browser tabs. - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/admin/workspace/index.md b/docs/automate/cmemc-command-line-interface/command-reference/admin/workspace/index.md index fdc9ae15..0c2bab3f 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/admin/workspace/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/admin/workspace/index.md @@ -5,7 +5,9 @@ icon: material/folder-multiple-outline tags: - cmemc --- + # admin workspace Command Group + Import, export and reload the project workspace. @@ -49,7 +51,7 @@ The file name is optional and will be generated with by the template if absent. Import the workspace from a file. ```shell-session title="Usage" -$ cmemc admin workspace import [OPTIONS] FILE +cmemc admin workspace import [OPTIONS] FILE ``` @@ -67,7 +69,7 @@ $ cmemc admin workspace import [OPTIONS] FILE Reload the workspace from the backend. ```shell-session title="Usage" -$ cmemc admin workspace reload +cmemc admin workspace reload ``` @@ -79,10 +81,9 @@ $ cmemc admin workspace reload Request a thread dump for all live threads. ```shell-session title="Usage" -$ cmemc admin workspace thread-dump +cmemc admin workspace thread-dump ``` - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/admin/workspace/python/index.md b/docs/automate/cmemc-command-line-interface/command-reference/admin/workspace/python/index.md index 8d83b58a..a5fb7bc0 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/admin/workspace/python/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/admin/workspace/python/index.md @@ -6,7 +6,9 @@ tags: - Python - cmemc --- + # admin workspace python Command Group + List, install, or uninstall python packages. @@ -23,7 +25,7 @@ Python packages are used to extend the Build (DataIntegration) workspace with py Install a python package to the workspace. ```shell-session title="Usage" -$ cmemc admin workspace python install PACKAGE +cmemc admin workspace python install PACKAGE ``` @@ -44,7 +46,7 @@ You can install a package by uploading a source distribution .tar.gz file, by up Uninstall a python packages from the workspace. ```shell-session title="Usage" -$ cmemc admin workspace python uninstall [OPTIONS] [PACKAGE_NAME]... +cmemc admin workspace python uninstall [OPTIONS] [PACKAGE_NAME]... ``` @@ -67,7 +69,7 @@ This command is essentially a `pip uninstall` in the remote python environment. List installed python packages. ```shell-session title="Usage" -$ cmemc admin workspace python list [OPTIONS] +cmemc admin workspace python list [OPTIONS] ``` @@ -96,7 +98,7 @@ It outputs a table of python package identifiers with version information. List installed workspace plugins. ```shell-session title="Usage" -$ cmemc admin workspace python list-plugins [OPTIONS] +cmemc admin workspace python list-plugins [OPTIONS] ``` @@ -123,7 +125,7 @@ This commands lists all discovered plugins. Open a package pypi.org page in the browser. ```shell-session title="Usage" -$ cmemc admin workspace python open PACKAGE +cmemc admin workspace python open PACKAGE ``` @@ -138,7 +140,7 @@ With this command, you can open the pypi.org page of a published package in your Reload / Register all installed plugins. ```shell-session title="Usage" -$ cmemc admin workspace python reload +cmemc admin workspace python reload ``` @@ -147,4 +149,3 @@ $ cmemc admin workspace python reload This command will register all installed plugins into the Build (DataIntegration) workspace. This command is useful, when you are installing packages into the Build Python environment without using the provided cmemc commands (e.g. by mounting a prepared filesystem in the docker container). - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/config/index.md b/docs/automate/cmemc-command-line-interface/command-reference/config/index.md index 07cb5031..25585fed 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/config/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/config/index.md @@ -6,7 +6,9 @@ tags: - Configuration - cmemc --- + # config Command Group + ```text @@ -64,7 +66,7 @@ manual. List configured connections. ```shell-session title="Usage" -$ cmemc config list [OPTIONS] +cmemc config list [OPTIONS] ``` @@ -79,17 +81,17 @@ You can use the `--filter` option to filter connections by regex matching the co In order to apply commands on more than one instance, you need to use typical unix gear such as xargs or parallel. ```shell-session title="Example" -$ cmemc config list +cmemc config list ``` ```shell-session title="Example" -$ cmemc config list --id-only | xargs -I % sh -c 'cmemc -c % admin status' +cmemc config list --id-only | xargs -I % sh -c 'cmemc -c % admin status' ``` ```shell-session title="Example" -$ cmemc config list --id-only | parallel --jobs 5 cmemc -c {} admin status +cmemc config list --id-only | parallel --jobs 5 cmemc -c {} admin status ``` @@ -109,7 +111,7 @@ $ cmemc config list --id-only | parallel --jobs 5 cmemc -c {} admin status Edit the user-scope configuration file. ```shell-session title="Usage" -$ cmemc config edit +cmemc config edit ``` @@ -133,7 +135,7 @@ $ cmemc config get {cmem_base_uri|ssl_verify|requests_ca_bundle|dp_api_end In order to automate processes such as fetching custom API data from multiple Corporate Memory instances, this command provides a way to get the value of a cmemc configuration key for the selected deployment. ```shell-session title="Example" -$ curl -H "Authorization: Bearer $(cmemc -c my admin token)" $(cmemc -c my config get DP_API_ENDPOINT)/api/custom/slug +curl -H "Authorization: Bearer $(cmemc -c my admin token)" $(cmemc -c my config get DP_API_ENDPOINT)/api/custom/slug ``` @@ -146,7 +148,7 @@ The commands return with exit code 1 if the config key is not used in the curren Export all configuration values of a configuration for evaluation. ```shell-session title="Usage" -$ cmemc config eval [OPTIONS] +cmemc config eval [OPTIONS] ``` @@ -155,7 +157,7 @@ $ cmemc config eval [OPTIONS] The output of this command is suitable to be used by a shell's `eval` command. It will output the complete configuration as `export key="value"` statements, which allow for the preparation of a shell environment. ```shell-session title="Example" -$ eval $(cmemc -c my config eval) +eval $(cmemc -c my config eval) ``` @@ -171,4 +173,3 @@ $ eval $(cmemc -c my config eval) --unset Instead of exporting all configuration keys, this option will unset all keys. ``` - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/dataset/index.md b/docs/automate/cmemc-command-line-interface/command-reference/dataset/index.md index 02d964cf..6c11c740 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/dataset/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/dataset/index.md @@ -5,7 +5,9 @@ icon: eccenca/artefact-dataset tags: - cmemc --- + # dataset Command Group + List, create, delete, inspect, up-/download or open datasets. @@ -265,4 +267,3 @@ $ cmemc dataset update my-project:my-csv -p separator ";" Note that this option already needs access to the instance. ``` - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/graph/imports/index.md b/docs/automate/cmemc-command-line-interface/command-reference/graph/imports/index.md index 19409ee6..c07d194e 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/graph/imports/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/graph/imports/index.md @@ -6,7 +6,9 @@ tags: - KnowledgeGraph - cmemc --- + # graph imports Command Group + List, create, delete and show graph imports. @@ -23,7 +25,7 @@ Graphs are identified by an IRI. Statement imports are managed by creating owl:i Show graph tree(s) of the imports statement hierarchy. ```shell-session title="Usage" -$ cmemc graph imports tree [OPTIONS] [IRIS]... +cmemc graph imports tree [OPTIONS] [IRIS]... ``` @@ -51,7 +53,7 @@ Imported graphs which do not exist are shown as `[missing: IRI]`. Imported graph List accessible graph imports statements. ```shell-session title="Usage" -$ cmemc graph imports list [OPTIONS] +cmemc graph imports list [OPTIONS] ``` @@ -74,7 +76,7 @@ Graphs are identified by an IRI. Statement imports are managed by creating owl:i Add statement to import a TO_GRAPH into a FROM_GRAPH. ```shell-session title="Usage" -$ cmemc graph imports create FROM_GRAPH TO_GRAPH +cmemc graph imports create FROM_GRAPH TO_GRAPH ``` @@ -93,7 +95,7 @@ Graphs are identified by an IRI. Statement imports are managed by creating owl:i Delete statement to import a TO_GRAPH into a FROM_GRAPH. ```shell-session title="Usage" -$ cmemc graph imports delete FROM_GRAPH TO_GRAPH +cmemc graph imports delete FROM_GRAPH TO_GRAPH ``` @@ -106,4 +108,3 @@ Graphs are identified by an IRI. Statement imports are managed by creating owl:i - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/graph/index.md b/docs/automate/cmemc-command-line-interface/command-reference/graph/index.md index 9db7cd46..58532e25 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/graph/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/graph/index.md @@ -6,7 +6,9 @@ tags: - KnowledgeGraph - cmemc --- + # graph Command Group + List, import, export, delete, count, tree or open graphs. @@ -23,7 +25,7 @@ Graphs are identified by an IRI. Count triples in graph(s). ```shell-session title="Usage" -$ cmemc graph count [OPTIONS] [IRIS]... +cmemc graph count [OPTIONS] [IRIS]... ``` @@ -45,7 +47,7 @@ This command lists graphs with their triple count. Counts do not include importe List accessible graphs. ```shell-session title="Usage" -$ cmemc graph list [OPTIONS] +cmemc graph list [OPTIONS] ``` @@ -114,7 +116,7 @@ In case of file export, data from all selected graphs will be concatenated in on Delete graph(s) from the store. ```shell-session title="Usage" -$ cmemc graph delete [OPTIONS] [IRIS]... +cmemc graph delete [OPTIONS] [IRIS]... ``` @@ -138,7 +140,7 @@ $ cmemc graph delete [OPTIONS] [IRIS]... Import graph(s) to the store. ```shell-session title="Usage" -$ cmemc graph import [OPTIONS] INPUT_PATH [IRI] +cmemc graph import [OPTIONS] INPUT_PATH [IRI] ``` @@ -178,10 +180,9 @@ If the ``--replace`` flag is set, the data in the graphs will be overwritten, if Open / explore a graph in the browser. ```shell-session title="Usage" -$ cmemc graph open IRI +cmemc graph open IRI ``` - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/graph/insights/index.md b/docs/automate/cmemc-command-line-interface/command-reference/graph/insights/index.md index 7452ef1e..f228757a 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/graph/insights/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/graph/insights/index.md @@ -5,7 +5,9 @@ icon: eccenca/graph-insights tags: - cmemc --- + # graph insights Command Group + List, create, delete and inspect graph insight snapshots. @@ -18,7 +20,7 @@ Graph Insight Snapshots are identified by an ID. To get a list of existing snaps List graph insight snapshots. ```shell-session title="Usage" -$ cmemc graph insights list [OPTIONS] +cmemc graph insights list [OPTIONS] ``` @@ -44,7 +46,7 @@ Graph Insights Snapshots are identified by an ID. Delete graph insight snapshots. ```shell-session title="Usage" -$ cmemc graph insights delete [OPTIONS] [SNAPSHOT_IDS]... +cmemc graph insights delete [OPTIONS] [SNAPSHOT_IDS]... ``` @@ -76,7 +78,7 @@ Graph Insight Snapshots are identified by an ID. Create or update a graph insight snapshot. ```shell-session title="Usage" -$ cmemc graph insights create [OPTIONS] IRI +cmemc graph insights create [OPTIONS] IRI ``` @@ -102,7 +104,7 @@ Create a graph insight snapshot for a given graph. If the snapshot already exist Update a graph insight snapshot. ```shell-session title="Usage" -$ cmemc graph insights update [OPTIONS] [SNAPSHOT_ID] +cmemc graph insights update [OPTIONS] [SNAPSHOT_ID] ``` @@ -133,7 +135,7 @@ After the update, the snapshot is hot-swapped. Inspect the metadata of a graph insight snapshot. ```shell-session title="Usage" -$ cmemc graph insights inspect [OPTIONS] SNAPSHOT_ID +cmemc graph insights inspect [OPTIONS] SNAPSHOT_ID ``` @@ -145,4 +147,3 @@ $ cmemc graph insights inspect [OPTIONS] SNAPSHOT_ID --raw Outputs raw JSON. ``` - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/graph/validation/index.md b/docs/automate/cmemc-command-line-interface/command-reference/graph/validation/index.md index 2932c48b..6994fc32 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/graph/validation/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/graph/validation/index.md @@ -7,7 +7,9 @@ tags: - Validation - cmemc --- + # graph validation Command Group + Validate resources in a graph. @@ -24,7 +26,7 @@ This command group is dedicated to the management of resource validation process Start a new validation process. ```shell-session title="Usage" -$ cmemc graph validation execute [OPTIONS] IRI +cmemc graph validation execute [OPTIONS] IRI ``` @@ -75,7 +77,7 @@ Validation is performed on all typed resources of the data / context graph (and List running and finished validation processes. ```shell-session title="Usage" -$ cmemc graph validation list [OPTIONS] +cmemc graph validation list [OPTIONS] ``` @@ -106,7 +108,7 @@ This command provides a filterable table or identifier list of validation proces List and inspect errors found with a validation process. ```shell-session title="Usage" -$ cmemc graph validation inspect [OPTIONS] PROCESS_ID +cmemc graph validation inspect [OPTIONS] PROCESS_ID ``` @@ -140,7 +142,7 @@ Use the ``--filter`` option to limit the output based on different criteria such Cancel a running validation process. ```shell-session title="Usage" -$ cmemc graph validation cancel PROCESS_ID +cmemc graph validation cancel PROCESS_ID ``` @@ -188,4 +190,3 @@ For jUnit XML: Each validation process result will be transformed to a single te --format [JSON|XML] Export either the plain JSON report or a distilled jUnit XML report. [default: XML] ``` - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/index.md b/docs/automate/cmemc-command-line-interface/command-reference/index.md index 3dd514fe..a9b6b37c 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/index.md @@ -6,7 +6,9 @@ tags: - Reference - cmemc --- + # Command Reference + !!! info @@ -88,6 +90,14 @@ tags: | [graph validation](graph/validation/index.md) | [inspect](graph/validation/index.md#graph-validation-inspect) | List and inspect errors found with a validation process. | | [graph validation](graph/validation/index.md) | [cancel](graph/validation/index.md#graph-validation-cancel) | Cancel a running validation process. | | [graph validation](graph/validation/index.md) | [export](graph/validation/index.md#graph-validation-export) | Export a report of finished validations. | +| [package](package/index.md) | [inspect](package/index.md#package-inspect) | Inspect the manifest of a package. | +| [package](package/index.md) | [list](package/index.md#package-list) | List installed packages. | +| [package](package/index.md) | [install](package/index.md#package-install) | Install packages. | +| [package](package/index.md) | [uninstall](package/index.md#package-uninstall) | Uninstall installed packages. | +| [package](package/index.md) | [export](package/index.md#package-export) | Export installed packages to package directories. | +| [package](package/index.md) | [build](package/index.md#package-build) | Build a package archive from a package directory. | +| [package](package/index.md) | [publish](package/index.md#package-publish) | Publish a package archive to the marketplace server. | +| [package](package/index.md) | [search](package/index.md#package-search) | Search for available packages with a given search text. | | [project](project/index.md) | [open](project/index.md#project-open) | Open projects in the browser. | | [project](project/index.md) | [list](project/index.md#project-list) | List available projects. | | [project](project/index.md) | [export](project/index.md#project-export) | Export projects to files. | @@ -133,4 +143,3 @@ tags: | [workflow scheduler](workflow/scheduler/index.md) | [inspect](workflow/scheduler/index.md#workflow-scheduler-inspect) | Display all metadata of a scheduler. | | [workflow scheduler](workflow/scheduler/index.md) | [disable](workflow/scheduler/index.md#workflow-scheduler-disable) | Disable scheduler(s). | | [workflow scheduler](workflow/scheduler/index.md) | [enable](workflow/scheduler/index.md#workflow-scheduler-enable) | Enable scheduler(s). | - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/package/index.md b/docs/automate/cmemc-command-line-interface/command-reference/package/index.md new file mode 100644 index 00000000..9176cbb5 --- /dev/null +++ b/docs/automate/cmemc-command-line-interface/command-reference/package/index.md @@ -0,0 +1,213 @@ +--- +title: "cmemc: Command Group - package" +description: "List, (un)install, export, create, or inspect packages." +icon: material/shopping +tags: + - cmemc + - Package +--- + +# package Command Group + + + +List, (un)install, export, create, or inspect packages. + + +## package inspect + +Inspect the manifest of a package. + +```shell-session title="Usage" +cmemc package inspect [OPTIONS] PACKAGE_PATH +``` + + + + + +??? info "Options" + ```text + + --key TEXT Get a specific key only from the manifest. + --raw Outputs raw JSON. + ``` + +## package list + +List installed packages. + +```shell-session title="Usage" +cmemc package list [OPTIONS] +``` + + + + + +??? info "Options" + ```text + + --filter ... Filter installed packages by one of the following + filter names and a corresponding value: type, name, + id. + --id-only Lists only package IDs. This is useful for piping + the IDs into other commands. + --raw Outputs raw JSON. + ``` + +## package install + +Install packages. + +```shell-session title="Usage" +cmemc package install [OPTIONS] [PACKAGE_ID] +``` + + + + +This command installs a package either from the marketplace or from local package archives (.cpa) or directories. + + + +??? info "Options" + ```text + + -i, --input PATH Install a package from a package archive (.cpa) or + directory. + --replace Replace (overwrite) existing package version, if + present. + --no-cache Disable using cached package versions. + --ignore-lock Ignore lock file for operation. This can be + dangerous, use with care. + --marketplace-url TEXT Base URL of the Marketplace - uses environment + variable ECCENCA_MARKETPLACE_URL if available. + [default: https://eccenca.market] + ``` + +## package uninstall + +Uninstall installed packages. + +```shell-session title="Usage" +cmemc package uninstall [OPTIONS] [PACKAGE_ID] +``` + + + + + +??? info "Options" + ```text + + --ignore-lock Ignore lock file for operation. This can be + dangerous, use with care. + --filter ... Filter installed packages by one of the following + filter names and a corresponding value: type, name, + id. + -a, --all Uninstall all packages. This is a dangerous option, + so use it with care. + ``` + +## package export + +Export installed packages to package directories. + +```shell-session title="Usage" +$ cmemc package export [OPTIONS] [PACKAGE_ID] +``` + + + + + +??? info "Options" + ```text + + --mime_type [text/turtle|text/turtle+pretty] + Choose the MIME type for graphs when + exporting packages. [default: + text/turtle+pretty] + --filter ... Filter installed packages by one of the + following filter names and a corresponding + value: type, name, id. + -a, --all Export all installed packages. + --output-dir DIRECTORY Create package directories in this base + directory. [default: .] + --replace Replace (overwrite) existing files, if + present. + ``` + +## package build + +Build a package archive from a package directory. + +```shell-session title="Usage" +cmemc package build [OPTIONS] PACKAGE_DIRECTORY +``` + + + + +This command processes a package directory, validates its content including the manifest, and creates a versioned Corporate Memory package archive (.cpa) with the following naming convention: {package_id}-v{version}.cpa + +Package archives can be published to the marketplace using the `package publish` command. + + + +??? info "Options" + ```text + + --version TEXT Set the package version. + --replace Replace package archive, if present. + --output-dir DIRECTORY Create the package archive in a specific directory. + [default: .] + ``` + +## package publish + +Publish a package archive to the marketplace server. + +```shell-session title="Usage" +cmemc package publish [OPTIONS] PACKAGE_ARCHIVE +``` + + + + + +??? info "Options" + ```text + + --timeout INTEGER Timeout for marketplace requests. + --marketplace-url TEXT Base URL of the Marketplace - uses environment + variable ECCENCA_MARKETPLACE_URL if available. + [default: https://eccenca.market] + --marketplace-account TEXT Marketplace account - uses environment variable + ECCENCA_MARKETPLACE_ACCOUNT if available. + --marketplace-password TEXT Marketplace password - uses environment + variable ECCENCA_MARKETPLACE_PASSWORD if + available. + ``` + +## package search + +Search for available packages with a given search text. + +```shell-session title="Usage" +cmemc package search [OPTIONS] [SEARCH_TERMS]... +``` + + + + + +??? info "Options" + ```text + + --raw Outputs raw JSON. + --marketplace-url TEXT Base URL of the Marketplace - uses environment + variable ECCENCA_MARKETPLACE_URL if available. + [default: https://eccenca.market] + ``` diff --git a/docs/automate/cmemc-command-line-interface/command-reference/project/file/index.md b/docs/automate/cmemc-command-line-interface/command-reference/project/file/index.md index efc4dd30..deecdd49 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/project/file/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/project/file/index.md @@ -6,7 +6,9 @@ tags: - Files - cmemc --- + # project file Command Group + List, inspect, up-/download or delete project file resources. @@ -23,7 +25,7 @@ File resources are identified with a `RESOURCE_ID` which is a concatenation of i List available file resources. ```shell-session title="Usage" -$ cmemc project file list [OPTIONS] +cmemc project file list [OPTIONS] ``` @@ -49,7 +51,7 @@ Outputs a table or a list of file resources. Delete file resources. ```shell-session title="Usage" -$ cmemc project file delete [OPTIONS] [RESOURCE_IDS]... +cmemc project file delete [OPTIONS] [RESOURCE_IDS]... ``` @@ -75,7 +77,7 @@ There are three selection mechanisms: with specific IDs - only those specified r Download file resources to the local file system. ```shell-session title="Usage" -$ cmemc project file download [OPTIONS] [RESOURCE_IDS]... +cmemc project file download [OPTIONS] [RESOURCE_IDS]... ``` @@ -86,12 +88,12 @@ This command downloads one or more file resources from projects to your local fi Resources are identified by their IDs in the format `PROJECT_ID`:`RESOURCE_NAME`. ```shell-session title="Example" -$ cmemc project file download my-proj:my-file.csv +cmemc project file download my-proj:my-file.csv ``` ```shell-session title="Example" -$ cmemc project file download my-proj:file1.csv my-proj:file2.csv --output-dir /tmp +cmemc project file download my-proj:file1.csv my-proj:file2.csv --output-dir /tmp ``` @@ -112,7 +114,7 @@ $ cmemc project file download my-proj:file1.csv my-proj:file2.csv --output-dir / Upload a file to a project. ```shell-session title="Usage" -$ cmemc project file upload [OPTIONS] INPUT_PATH +cmemc project file upload [OPTIONS] INPUT_PATH ``` @@ -125,7 +127,7 @@ This command uploads a file to a project as a file resource. ```shell-session title="Example" -$ cmemc project file upload my-file.csv --project my-project +cmemc project file upload my-file.csv --project my-project ``` @@ -149,7 +151,7 @@ $ cmemc project file upload my-file.csv --project my-project Display all metadata of a file resource. ```shell-session title="Usage" -$ cmemc project file inspect [OPTIONS] RESOURCE_ID +cmemc project file inspect [OPTIONS] RESOURCE_ID ``` @@ -167,7 +169,7 @@ $ cmemc project file inspect [OPTIONS] RESOURCE_ID Display all usage data of a file resource. ```shell-session title="Usage" -$ cmemc project file usage [OPTIONS] RESOURCE_ID +cmemc project file usage [OPTIONS] RESOURCE_ID ``` @@ -179,4 +181,3 @@ $ cmemc project file usage [OPTIONS] RESOURCE_ID --raw Outputs raw JSON. ``` - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/project/index.md b/docs/automate/cmemc-command-line-interface/command-reference/project/index.md index 33c95fe3..22b732e3 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/project/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/project/index.md @@ -6,7 +6,9 @@ tags: - Project - cmemc --- + # project Command Group + List, import, export, create, delete or open projects. @@ -23,7 +25,7 @@ Projects are identified by a `PROJECT_ID`. Open projects in the browser. ```shell-session title="Usage" -$ cmemc project open PROJECT_IDS... +cmemc project open PROJECT_IDS... ``` @@ -40,7 +42,7 @@ The command accepts multiple project IDs which results in opening multiple brows List available projects. ```shell-session title="Usage" -$ cmemc project list [OPTIONS] +cmemc project list [OPTIONS] ``` @@ -122,14 +124,14 @@ $ cmemc config list | parallel -I% cmemc -c % project export --all -t "dump/{{co Import a project from a file or directory. ```shell-session title="Usage" -$ cmemc project import [OPTIONS] PATH [PROJECT_ID] +cmemc project import [OPTIONS] PATH [PROJECT_ID] ``` ```shell-session title="Example" -$ cmemc project import my_project.zip my_project +cmemc project import my_project.zip my_project ``` @@ -147,7 +149,7 @@ $ cmemc project import my_project.zip my_project Delete projects. ```shell-session title="Usage" -$ cmemc project delete [OPTIONS] [PROJECT_IDS]... +cmemc project delete [OPTIONS] [PROJECT_IDS]... ``` @@ -179,7 +181,7 @@ This command deletes existing data integration projects from Corporate Memory. Create projects. ```shell-session title="Usage" -$ cmemc project create [OPTIONS] PROJECT_IDS... +cmemc project create [OPTIONS] PROJECT_IDS... ``` @@ -208,7 +210,7 @@ This command creates one or more new projects. Existing projects will not be ove Reload projects from the workspace provider. ```shell-session title="Usage" -$ cmemc project reload [OPTIONS] [PROJECT_IDS]... +cmemc project reload [OPTIONS] [PROJECT_IDS]... ``` @@ -231,4 +233,3 @@ This command reloads all tasks of a project from the workspace provider. This is -a, --all Reload all projects ``` - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/project/variable/index.md b/docs/automate/cmemc-command-line-interface/command-reference/project/variable/index.md index 3c870025..45c9ed5b 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/project/variable/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/project/variable/index.md @@ -6,7 +6,9 @@ tags: - Variables - cmemc --- + # project variable Command Group + List, create, delete or get data from project variables. @@ -21,7 +23,7 @@ Variables are identified by a `VARIABLE_ID`. To get a list of existing variables List available project variables. ```shell-session title="Usage" -$ cmemc project variable list [OPTIONS] +cmemc project variable list [OPTIONS] ``` @@ -48,7 +50,7 @@ Outputs a table or a list of project variables. Get the value or other data of a project variable. ```shell-session title="Usage" -$ cmemc project variable get [OPTIONS] VARIABLE_ID +cmemc project variable get [OPTIONS] VARIABLE_ID ``` @@ -76,7 +78,7 @@ Use the ``--key`` option to specify which information you want to get. Delete project variables. ```shell-session title="Usage" -$ cmemc project variable delete [OPTIONS] [VARIABLE_IDS]... +cmemc project variable delete [OPTIONS] [VARIABLE_IDS]... ``` @@ -103,7 +105,7 @@ Variables are automatically sorted by their dependencies and deleted in the corr Create a new project variable. ```shell-session title="Usage" -$ cmemc project variable create [OPTIONS] VARIABLE_NAME +cmemc project variable create [OPTIONS] VARIABLE_NAME ``` @@ -112,7 +114,7 @@ $ cmemc project variable create [OPTIONS] VARIABLE_NAME Variables need to be created with a value or a template (not both). In addition to that, a project ID and a name are mandatory. ```shell-session title="Example" -$ cmemc project variable create my_var --project my_project --value abc +cmemc project variable create my_var --project my_project --value abc ``` @@ -141,7 +143,7 @@ $ cmemc project variable create my_var --project my_project --value abc Update data of an existing project variable. ```shell-session title="Usage" -$ cmemc project variable update [OPTIONS] VARIABLE_ID +cmemc project variable update [OPTIONS] VARIABLE_ID ``` @@ -165,4 +167,3 @@ With this command you can update the value or the template, as well as the descr accessing variables from the same project. --description TEXT The new description of the project variable. ``` - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/query/index.md b/docs/automate/cmemc-command-line-interface/command-reference/query/index.md index ea6f5a6a..dde3a687 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/query/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/query/index.md @@ -6,7 +6,9 @@ tags: - SPARQL - cmemc --- + # query Command Group + List, execute, get status or open SPARQL queries. @@ -27,7 +29,7 @@ Queries can use a mustache like syntax to specify placeholder for parameter valu Execute queries which are loaded from files or a query catalog. ```shell-session title="Usage" -$ cmemc query execute [OPTIONS] QUERIES... +cmemc query execute [OPTIONS] QUERIES... ``` @@ -84,7 +86,7 @@ Limitations: All optional parameters (e.g. accept, base64, ...) are provided for List available queries from a query catalog. ```shell-session title="Usage" -$ cmemc query list [OPTIONS] +cmemc query list [OPTIONS] ``` @@ -114,7 +116,7 @@ You can filter queries based on ID, type, placeholder, or regex pattern. Open queries in the editor of a query catalog in your browser. ```shell-session title="Usage" -$ cmemc query open [OPTIONS] QUERIES... +cmemc query open [OPTIONS] QUERIES... ``` @@ -138,7 +140,7 @@ The command accepts multiple query URIs or files which results in opening multip Get status information of executed and running queries. ```shell-session title="Usage" -$ cmemc query status [OPTIONS] [QUERY_ID] +cmemc query status [OPTIONS] [QUERY_ID] ``` @@ -167,7 +169,7 @@ You can filter queries based on status and runtime in order to investigate slow Re-execute queries from a replay file. ```shell-session title="Usage" -$ cmemc query replay [OPTIONS] REPLAY_FILE +cmemc query replay [OPTIONS] REPLAY_FILE ``` @@ -176,7 +178,7 @@ $ cmemc query replay [OPTIONS] REPLAY_FILE This command reads a `REPLAY_FILE` and re-executes the logged queries. A `REPLAY_FILE` is a JSON document which is an array of JSON objects with at least a key `queryString` holding the query text OR a key `iri` holding the IRI of the query in the query catalog. It can be created with the `query status` command. ```shell-session title="Example" -$ query status --raw > replay.json +query status --raw > replay.json ``` @@ -208,7 +210,7 @@ The optional output file is the same JSON document which is used as input, but e Cancel a running query. ```shell-session title="Usage" -$ cmemc query cancel QUERY_ID +cmemc query cancel QUERY_ID ``` @@ -223,7 +225,7 @@ With this command, you can cancel a running query. Depending on the backend trip Explain queries by showing their logical execution plan. ```shell-session title="Usage" -$ cmemc query explain [OPTIONS] QUERIES... +cmemc query explain [OPTIONS] QUERIES... ``` @@ -256,7 +258,7 @@ This helps understand query performance and identify optimization opportunities. Create a new query in a query catalog. ```shell-session title="Usage" -$ cmemc query create [OPTIONS] QUERY_FILE +cmemc query create [OPTIONS] QUERY_FILE ``` @@ -267,7 +269,7 @@ Creates a new query in a query catalog from a SPARQL query file. The query type (SELECT, CONSTRUCT, UPDATE, etc.) is automatically detected from the query text. ```shell-session title="Example" -$ cmemc query create my-query.sparql $ cmemc query create my-query.sparql --id :customName +cmemc query create my-query.sparql $ cmemc query create my-query.sparql --id :customName ``` @@ -292,7 +294,7 @@ $ cmemc query create my-query.sparql $ cmemc query create my-query.sparql --id : Update an existing query in a query catalog. ```shell-session title="Usage" -$ cmemc query update [OPTIONS] QUERY_ID +cmemc query update [OPTIONS] QUERY_ID ``` @@ -319,7 +321,7 @@ The `QUERY_ID` can be either a short URI (e.g., :myQuery) or a full URI. Delete queries from a query catalog. ```shell-session title="Usage" -$ cmemc query delete [OPTIONS] [QUERY_IDS]... +cmemc query delete [OPTIONS] [QUERY_IDS]... ``` @@ -346,4 +348,3 @@ You can delete specific queries by providing their IDs, or use `--filter` to sel --all Delete all queries from the catalog. Use with caution! ``` - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/vocabulary/cache/index.md b/docs/automate/cmemc-command-line-interface/command-reference/vocabulary/cache/index.md index fa503b08..76ec444a 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/vocabulary/cache/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/vocabulary/cache/index.md @@ -6,7 +6,9 @@ tags: - Vocabulary - cmemc --- + # vocabulary cache Command Group + List und update the vocabulary cache. @@ -17,7 +19,7 @@ List und update the vocabulary cache. Reload / updates the data integration cache for a vocabulary. ```shell-session title="Usage" -$ cmemc vocabulary cache update [OPTIONS] [IRIS]... +cmemc vocabulary cache update [OPTIONS] [IRIS]... ``` @@ -35,7 +37,7 @@ $ cmemc vocabulary cache update [OPTIONS] [IRIS]... Output the content of the global vocabulary cache. ```shell-session title="Usage" -$ cmemc vocabulary cache list [OPTIONS] +cmemc vocabulary cache list [OPTIONS] ``` @@ -50,4 +52,3 @@ $ cmemc vocabulary cache list [OPTIONS] cmemc commands. --raw Outputs raw JSON. ``` - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/vocabulary/index.md b/docs/automate/cmemc-command-line-interface/command-reference/vocabulary/index.md index e09e522e..3dc93018 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/vocabulary/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/vocabulary/index.md @@ -6,7 +6,9 @@ tags: - Vocabulary - cmemc --- + # vocabulary Command Group + List, (un-)install, import or open vocabs / manage cache. @@ -17,7 +19,7 @@ List, (un-)install, import or open vocabs / manage cache. Open / explore a vocabulary graph in the browser. ```shell-session title="Usage" -$ cmemc vocabulary open IRI +cmemc vocabulary open IRI ``` @@ -32,7 +34,7 @@ Vocabularies are identified by their graph IRI. Installed vocabularies can be li Output a list of vocabularies. ```shell-session title="Usage" -$ cmemc vocabulary list [OPTIONS] +cmemc vocabulary list [OPTIONS] ``` @@ -60,7 +62,7 @@ Vocabularies are graphs (see `graph` command group) which consists of class and Install one or more vocabularies from the catalog. ```shell-session title="Usage" -$ cmemc vocabulary install [OPTIONS] [IRIS]... +cmemc vocabulary install [OPTIONS] [IRIS]... ``` @@ -82,7 +84,7 @@ Vocabularies are identified by their graph IRI. Installable vocabularies can be Uninstall one or more vocabularies. ```shell-session title="Usage" -$ cmemc vocabulary uninstall [OPTIONS] [IRIS]... +cmemc vocabulary uninstall [OPTIONS] [IRIS]... ``` @@ -103,7 +105,7 @@ Vocabularies are identified by their graph IRI. Already installed vocabularies c Import a turtle file as a vocabulary. ```shell-session title="Usage" -$ cmemc vocabulary import [OPTIONS] FILE +cmemc vocabulary import [OPTIONS] FILE ``` @@ -126,4 +128,3 @@ The uploaded ontology file is analysed locally in order to discover the named gr --replace Replace (overwrite) existing vocabulary, if present. ``` - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/workflow/index.md b/docs/automate/cmemc-command-line-interface/command-reference/workflow/index.md index 139d7d6a..1684ee74 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/workflow/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/workflow/index.md @@ -6,7 +6,9 @@ tags: - Workflow - cmemc --- + # workflow Command Group + List, execute, status or open (io) workflows. @@ -19,7 +21,7 @@ Workflows are identified by a `WORKFLOW_ID`. The get a list of existing workflow Execute workflow(s). ```shell-session title="Usage" -$ cmemc workflow execute [OPTIONS] [WORKFLOW_IDS]... +cmemc workflow execute [OPTIONS] [WORKFLOW_IDS]... ``` @@ -52,7 +54,7 @@ The optional `--wait` option starts the workflows in the same way, but also poll Execute a workflow with file input/output. ```shell-session title="Usage" -$ cmemc workflow io [OPTIONS] WORKFLOW_ID +cmemc workflow io [OPTIONS] WORKFLOW_ID ``` @@ -101,7 +103,7 @@ With this command, you can execute a workflow that uses replaceable datasets as List available workflow. ```shell-session title="Usage" -$ cmemc workflow list [OPTIONS] +cmemc workflow list [OPTIONS] ``` @@ -127,7 +129,7 @@ $ cmemc workflow list [OPTIONS] Get status information of workflow(s). ```shell-session title="Usage" -$ cmemc workflow status [OPTIONS] [WORKFLOW_IDS]... +cmemc workflow status [OPTIONS] [WORKFLOW_IDS]... ``` @@ -150,10 +152,9 @@ $ cmemc workflow status [OPTIONS] [WORKFLOW_IDS]... Open a workflow in your browser. ```shell-session title="Usage" -$ cmemc workflow open WORKFLOW_ID +cmemc workflow open WORKFLOW_ID ``` - diff --git a/docs/automate/cmemc-command-line-interface/command-reference/workflow/scheduler/index.md b/docs/automate/cmemc-command-line-interface/command-reference/workflow/scheduler/index.md index f868a14f..9c2bfb15 100644 --- a/docs/automate/cmemc-command-line-interface/command-reference/workflow/scheduler/index.md +++ b/docs/automate/cmemc-command-line-interface/command-reference/workflow/scheduler/index.md @@ -6,7 +6,9 @@ tags: - Automate - cmemc --- + # workflow scheduler Command Group + List, inspect, enable/disable or open scheduler. @@ -19,7 +21,7 @@ Schedulers execute workflows in specified intervals. They are identified with a Open scheduler(s) in the browser. ```shell-session title="Usage" -$ cmemc workflow scheduler open [OPTIONS] SCHEDULER_IDS... +cmemc workflow scheduler open [OPTIONS] SCHEDULER_IDS... ``` @@ -43,7 +45,7 @@ The command accepts multiple scheduler IDs which results in opening multiple bro List available scheduler. ```shell-session title="Usage" -$ cmemc workflow scheduler list [OPTIONS] +cmemc workflow scheduler list [OPTIONS] ``` @@ -66,7 +68,7 @@ Outputs a table or a list of scheduler IDs which can be used as reference for th Display all metadata of a scheduler. ```shell-session title="Usage" -$ cmemc workflow scheduler inspect [OPTIONS] SCHEDULER_ID +cmemc workflow scheduler inspect [OPTIONS] SCHEDULER_ID ``` @@ -84,7 +86,7 @@ $ cmemc workflow scheduler inspect [OPTIONS] SCHEDULER_ID Disable scheduler(s). ```shell-session title="Usage" -$ cmemc workflow scheduler disable [OPTIONS] [SCHEDULER_IDS]... +cmemc workflow scheduler disable [OPTIONS] [SCHEDULER_IDS]... ``` @@ -105,7 +107,7 @@ The command accepts multiple scheduler IDs which results in disabling them one a Enable scheduler(s). ```shell-session title="Usage" -$ cmemc workflow scheduler enable [OPTIONS] [SCHEDULER_IDS]... +cmemc workflow scheduler enable [OPTIONS] [SCHEDULER_IDS]... ``` @@ -120,4 +122,3 @@ The command accepts multiple scheduler IDs which results in enabling them one af -a, --all Enable all scheduler. ``` - diff --git a/docs/automate/cmemc-command-line-interface/configuration/certificate-handling-and-ssl-verification/index.md b/docs/automate/cmemc-command-line-interface/configuration/certificate-handling-and-ssl-verification/index.md index cc2a3bec..ae361974 100644 --- a/docs/automate/cmemc-command-line-interface/configuration/certificate-handling-and-ssl-verification/index.md +++ b/docs/automate/cmemc-command-line-interface/configuration/certificate-handling-and-ssl-verification/index.md @@ -64,9 +64,9 @@ miGId7jMXd24bpfYZSiniC0+SHiCwEmzN818Ss9aIMChymAnV3RRB/UqKLlOMnA= You can also disable SSL Verification completely by setting the `SSL_VERIFY` key in the config or environment to `false`. However, this will lead to warnings: + ``` shell-session $ cmemc -c ssltest.eccenca.com graph list SSL verification is disabled (SSL_VERIFY=False). ... ``` - diff --git a/docs/automate/cmemc-command-line-interface/configuration/completion-setup/index.md b/docs/automate/cmemc-command-line-interface/configuration/completion-setup/index.md index 15565537..127f566f 100644 --- a/docs/automate/cmemc-command-line-interface/configuration/completion-setup/index.md +++ b/docs/automate/cmemc-command-line-interface/configuration/completion-setup/index.md @@ -26,25 +26,22 @@ We suggest using [zsh](https://en.wikipedia.org/wiki/Z_shell) so you can take ad Use the following lines for the completion setup of cmemc >= 23.3. If using an older version, look at the [old documenation](https://documentation.eccenca.com/23.1/automate/cmemc-command-line-interface/configuration/completion-setup/). - In order to enable tab completion with **zsh** run the following command: ``` shell-session title="completion setup for zsh" -$ eval "$(_CMEMC_COMPLETE=zsh_source cmemc)" +eval "$(_CMEMC_COMPLETE=zsh_source cmemc)" ``` To enable the interactive menu as seen above in **zsh** run the following command: ``` shell-session title="interactive menu for zsh" -$ zstyle ':completion:*' menu select +zstyle ':completion:*' menu select ``` In order to enable tab completion with **bash** run the following command: ``` shell-session title="completion setup for bash" -$ eval "$(_CMEMC_COMPLETE=bash_source cmemc)" +eval "$(_CMEMC_COMPLETE=bash_source cmemc)" ``` You may want to add this line to your `.bashrc` or `.zshrc`. - - diff --git a/docs/automate/cmemc-command-line-interface/configuration/environment-based-configuration/index.md b/docs/automate/cmemc-command-line-interface/configuration/environment-based-configuration/index.md index 4d1fa12d..b70ee965 100644 --- a/docs/automate/cmemc-command-line-interface/configuration/environment-based-configuration/index.md +++ b/docs/automate/cmemc-command-line-interface/configuration/environment-based-configuration/index.md @@ -27,10 +27,10 @@ For these variables the rules are simple: You can use any variable from the [con The following commands provide the same result as given in the [basic example for a config file](../file-based-configuration/index.md): ``` shell-session -$ export CMEM_BASE_URI=http://localhost/ -$ export OAUTH_GRANT_TYPE=client_credentials -$ export OAUTH_CLIENT_ID=cmem-service-account -$ export OAUTH_CLIENT_SECRET=... +export CMEM_BASE_URI=http://localhost/ +export OAUTH_GRANT_TYPE=client_credentials +export OAUTH_CLIENT_ID=cmem-service-account +export OAUTH_CLIENT_SECRET=... ``` !!! info @@ -63,8 +63,8 @@ $ cmemc --config-file cmemc.ini --connection mycmem graph list --raw As a next step, we replace all connection parameters with environment variables: ``` shell-session -$ export CMEMC_CONFIG_FILE=cmemc.ini -$ export CMEMC_CONNECTION=mycmem +export CMEMC_CONFIG_FILE=cmemc.ini +export CMEMC_CONNECTION=mycmem ``` This alone allows us to save a lot of typing for a series of commands on the same Corporate Memory instance. @@ -77,7 +77,7 @@ $ cmemc graph list --raw However, you can also pre-define command options in the same way: ``` shell-session -$ export CMEMC_GRAPH_LIST_RAW=true +export CMEMC_GRAPH_LIST_RAW=true ``` Again, the same command but `--raw` is set per default. @@ -92,7 +92,7 @@ $ cmemc graph list Since there is a top level `--debug` option, the corresponding variable name is `CMEMC_DEBUG`: ``` shell-session -$ export CMEMC_DEBUG=true +export CMEMC_DEBUG=true ``` ## Configuration environment export from the config file @@ -118,12 +118,12 @@ export SSL_VERIFY="True" This can be used to export a full `config.env` or to `eval` it in an environment for other processes: ``` shell-session -$ cmemc -c my-cmem.example.org config eval > config.env -$ eval $(cmemc -c my-cmem.example.org config eval) +cmemc -c my-cmem.example.org config eval > config.env +eval $(cmemc -c my-cmem.example.org config eval) ``` Please note that the following command has the same effect but needs the `cmemc.ini` for evaluating the `config` values for the config section `my-cmem.example.org`: ``` shell-session -$ export CMEMC_CONNECTION="my-cmem.example.org" +export CMEMC_CONNECTION="my-cmem.example.org" ``` diff --git a/docs/automate/cmemc-command-line-interface/configuration/file-based-configuration/index.md b/docs/automate/cmemc-command-line-interface/configuration/file-based-configuration/index.md index a152b457..6f4eb533 100644 --- a/docs/automate/cmemc-command-line-interface/configuration/file-based-configuration/index.md +++ b/docs/automate/cmemc-command-line-interface/configuration/file-based-configuration/index.md @@ -26,7 +26,6 @@ If you need to change this location and want to use another config file, you hav However, once you start cmemc the first time without any command or option, it will create an empty configuration file at this location and will output a general introduction. - ??? example "First cmemc run ..." ``` shell-session $ cmemc @@ -270,4 +269,3 @@ Setting this to a PEM file allows for using private Certificate Authorities for Please refer to [Certificate handling and SSL verification](../certificate-handling-and-ssl-verification/index.md) for more information. This variable defaults to `$PYTHON_HOME/site-packages/certifi/cacert.pem`. - diff --git a/docs/automate/cmemc-command-line-interface/configuration/getting-credentials-from-external-processes/index.md b/docs/automate/cmemc-command-line-interface/configuration/getting-credentials-from-external-processes/index.md index 6ae2a4cc..1995b8b9 100644 --- a/docs/automate/cmemc-command-line-interface/configuration/getting-credentials-from-external-processes/index.md +++ b/docs/automate/cmemc-command-line-interface/configuration/getting-credentials-from-external-processes/index.md @@ -20,11 +20,11 @@ As described in the [Configuration with Environment Variables](../environment-ba The following code snippet demonstrates the behaviour: ``` shell-session -$ export CMEM_BASE_URI="https://your-cmem.eccenca.dev/" -$ export OAUTH_GRANT_TYPE="client_credentials" -$ export OAUTH_CLIENT_ID="cmem-service-account" -$ export OAUTH_CLIENT_SECRET="...secret..." -$ cmemc graph list +export CMEM_BASE_URI="https://your-cmem.eccenca.dev/" +export OAUTH_GRANT_TYPE="client_credentials" +export OAUTH_CLIENT_ID="cmem-service-account" +export OAUTH_CLIENT_SECRET="...secret..." +cmemc graph list ``` In the context of a CI/CD pipeline, e.g., on github, these credentials can be taken from the repository secrets: @@ -48,7 +48,7 @@ jobs: In shell context, you can fetch the secret from an external process to the variable: ``` shell-session -$ export OAUTH_CLIENT_SECRET=$(get-my-secret.sh) +export OAUTH_CLIENT_SECRET=$(get-my-secret.sh) ``` ## External Processes @@ -116,4 +116,3 @@ if [ "${OAUTH_GRANT_TYPE}" = "password" ]; then fi exit 1 ``` - diff --git a/docs/automate/cmemc-command-line-interface/configuration/index.md b/docs/automate/cmemc-command-line-interface/configuration/index.md index b6b25c39..8dce0e91 100644 --- a/docs/automate/cmemc-command-line-interface/configuration/index.md +++ b/docs/automate/cmemc-command-line-interface/configuration/index.md @@ -10,28 +10,27 @@ hide: In order to work with cmemc, you have to configure it according to your needs. -
-- :material-file-cog-outline: File-based Configuration +- :material-file-cog-outline: File-based Configuration --- The most common way to configure cmemc is with a central [configuration file](file-based-configuration/index.md). -- :material-cog-outline: Environment-based Configuration +- :material-cog-outline: Environment-based Configuration --- In addition to configuration files, cmemc can be widely configured and parameterized with [environment variables](environment-based-configuration/index.md). -- :material-rocket-launch: Completion Setup +- :material-rocket-launch: Completion Setup --- Setting up [command completion](completion-setup/index.md) is optional but highly recommended and will greatly speed up your cmemc terminal sessions. -- :material-key-link: Security Considerations +- :material-key-link: Security Considerations --- diff --git a/docs/automate/cmemc-command-line-interface/index.md b/docs/automate/cmemc-command-line-interface/index.md index f54bb65f..b39468a9 100644 --- a/docs/automate/cmemc-command-line-interface/index.md +++ b/docs/automate/cmemc-command-line-interface/index.md @@ -13,7 +13,7 @@ tags:
-- :octicons-terminal-16: **Command Line** interface for **eccenca Corporate Memory** +- :octicons-terminal-16: **Command Line** interface for **eccenca Corporate Memory** --- @@ -33,7 +33,7 @@ tags: [![pypy downloads](https://img.shields.io/pypi/dm/cmem-cmemc.svg "pypy downloads"){ .off-glb }](https://pypi.python.org/pypi/cmem-cmemc/) [![Docker Image](https://img.shields.io/badge/docker-image-blue?logo=docker&logoColor=white "Docker Image"){ .off-glb }](./invocation/docker-image/index.md) -- :octicons-people-24: Intended for **Administrators** and **Linked Data Expert** +- :octicons-people-24: Intended for **Administrators** and **Linked Data Expert** --- @@ -46,13 +46,12 @@ tags: --filter tag velocity-daily ``` - 1. :person_raising_hand: + 1. :person_raising_hand: - The option `-c` is short for `--connection` and references to a remote Corporate Memory instance. - The `list` command in the `dataset` command group shows all datasets of an instance. - In order to manipulate output dataset list, the `--filter` option takes two parameter, a filter type (`tag`, `project`, ...) and a value. - -- :octicons-rocket-16: Fast ad-hoc Execution with **Command Completion** +- :octicons-rocket-16: Fast ad-hoc Execution with **Command Completion** --- @@ -61,8 +60,7 @@ tags:
Create Build Project and Dataset
- -- :material-feature-search-outline: **Main Features**: +- :material-feature-search-outline: **Main Features**: --- @@ -76,4 +74,3 @@ tags: ```
- diff --git a/docs/automate/cmemc-command-line-interface/installation/index.md b/docs/automate/cmemc-command-line-interface/installation/index.md index d36233d2..51cc27a0 100644 --- a/docs/automate/cmemc-command-line-interface/installation/index.md +++ b/docs/automate/cmemc-command-line-interface/installation/index.md @@ -13,16 +13,13 @@ cmemc can be installed using the python package from pypi.org, the release packa cmemc is available as an [official pypi package](https://pypi.org/project/cmem-cmemc/) so installation can be done with pip or pipx (preferred): ``` shell-session -$ pipx install cmem-cmemc +pipx install cmem-cmemc ``` - ## ... via docker image This topic is described on a [stand-alone page](../invocation/docker-image/index.md). - !!! Note Once you have installed cmemc, you need to configure a connection with a [config file](../configuration/file-based-configuration/index.md) or learn how to [use environment variables](../configuration/environment-based-configuration/index.md) to control cmemc. - diff --git a/docs/automate/cmemc-command-line-interface/invocation/docker-image/index.md b/docs/automate/cmemc-command-line-interface/invocation/docker-image/index.md index d309bafa..8a6ae762 100644 --- a/docs/automate/cmemc-command-line-interface/invocation/docker-image/index.md +++ b/docs/automate/cmemc-command-line-interface/invocation/docker-image/index.md @@ -67,4 +67,3 @@ http://schema.org/,8809 https://vocab.eccenca.com/shacl/,1752 [...] ``` - diff --git a/docs/automate/cmemc-command-line-interface/invocation/github-action/index.md b/docs/automate/cmemc-command-line-interface/invocation/github-action/index.md index 54c3264f..0982159a 100644 --- a/docs/automate/cmemc-command-line-interface/invocation/github-action/index.md +++ b/docs/automate/cmemc-command-line-interface/invocation/github-action/index.md @@ -60,4 +60,3 @@ The Github project [eccenca/cmemc-workflow](https://github.com/eccenca/cmemc-wor Here is an example output: ![Example workflow output](example-workflow-output.png "Example workflow output") - diff --git a/docs/automate/cmemc-command-line-interface/invocation/gitlab-pipeline/index.md b/docs/automate/cmemc-command-line-interface/invocation/gitlab-pipeline/index.md index a40a0199..cadc5bd9 100644 --- a/docs/automate/cmemc-command-line-interface/invocation/gitlab-pipeline/index.md +++ b/docs/automate/cmemc-command-line-interface/invocation/gitlab-pipeline/index.md @@ -61,4 +61,3 @@ The Github project [eccenca/cmemc-workflow](https://github.com/eccenca/cmemc-wor Here is an example output: ![Example pipeline output](example-pipeline-output.png "Example pipeline output") - diff --git a/docs/automate/cmemc-command-line-interface/invocation/index.md b/docs/automate/cmemc-command-line-interface/invocation/index.md index 418dd4a9..bdf6f2e9 100644 --- a/docs/automate/cmemc-command-line-interface/invocation/index.md +++ b/docs/automate/cmemc-command-line-interface/invocation/index.md @@ -12,12 +12,12 @@ Besides the plain ad-hoc invocation from a users terminal, the following recipes
-- :material-docker: Executing cmemc as a [Docker Container](docker-image/index.md). +- :material-docker: Executing cmemc as a [Docker Container](docker-image/index.md). -- :material-github: Running cmemc jobs as part of [Github Actions](github-action/index.md). +- :material-github: Running cmemc jobs as part of [Github Actions](github-action/index.md). -- :material-gitlab: Running cmemc jobs as part of [Gitlab Pipelines](gitlab-pipeline/index.md). +- :material-gitlab: Running cmemc jobs as part of [Gitlab Pipelines](gitlab-pipeline/index.md). -- :eccenca-application-queries: Preparing [SPARQL Scripts](sparql-scripts/index.md) to fetch data from your Knowledge Graphs. -
+- :eccenca-application-queries: Preparing [SPARQL Scripts](sparql-scripts/index.md) to fetch data from your Knowledge Graphs. +
diff --git a/docs/automate/cmemc-command-line-interface/invocation/sparql-scripts/index.md b/docs/automate/cmemc-command-line-interface/invocation/sparql-scripts/index.md index dc31287c..f4315b00 100644 --- a/docs/automate/cmemc-command-line-interface/invocation/sparql-scripts/index.md +++ b/docs/automate/cmemc-command-line-interface/invocation/sparql-scripts/index.md @@ -34,7 +34,7 @@ This will set cmemc as an interpreter for the rest of the file, and by using the Now you need to define your SPARQL file as executable and run it: ``` shell-session -$ chmod a+x ./count-graphs.sh +chmod a+x ./count-graphs.sh ``` ``` shell-session @@ -48,4 +48,3 @@ https://ns.eccenca.com/data/queries/,39 https://ns.eccenca.com/data/config/,4 https://ns.eccenca.com/data/userinfo/,4 ``` - diff --git a/docs/automate/cmemc-command-line-interface/troubleshooting-and-caveats/index.md b/docs/automate/cmemc-command-line-interface/troubleshooting-and-caveats/index.md index 4b103789..75c3f6aa 100644 --- a/docs/automate/cmemc-command-line-interface/troubleshooting-and-caveats/index.md +++ b/docs/automate/cmemc-command-line-interface/troubleshooting-and-caveats/index.md @@ -36,4 +36,3 @@ This can have multiple reasons - please check in the following order: - `application.yaml` of DataIntegration - reverse proxy configuration - diff --git a/docs/automate/cmemc-command-line-interface/workflow-execution-and-orchestration/index.md b/docs/automate/cmemc-command-line-interface/workflow-execution-and-orchestration/index.md index 95c3a61e..c271f411 100644 --- a/docs/automate/cmemc-command-line-interface/workflow-execution-and-orchestration/index.md +++ b/docs/automate/cmemc-command-line-interface/workflow-execution-and-orchestration/index.md @@ -147,4 +147,3 @@ else exit 0 fi ``` - diff --git a/docs/automate/continuous-integration/index.md b/docs/automate/continuous-integration/index.md index ed967e2c..c7780dc0 100644 --- a/docs/automate/continuous-integration/index.md +++ b/docs/automate/continuous-integration/index.md @@ -33,9 +33,8 @@ The following pages provide recipes for different CI/CD solutions:
-- :material-github: [Github Actions](../cmemc-command-line-interface/invocation/github-action/index.md) +- :material-github: [Github Actions](../cmemc-command-line-interface/invocation/github-action/index.md) -- :material-gitlab: [Gitlab Pipelines](../cmemc-command-line-interface/invocation/gitlab-pipeline/index.md) +- :material-gitlab: [Gitlab Pipelines](../cmemc-command-line-interface/invocation/gitlab-pipeline/index.md)
- diff --git a/docs/automate/index.md b/docs/automate/index.md index 48dc8623..f7ff7c04 100644 --- a/docs/automate/index.md +++ b/docs/automate/index.md @@ -11,29 +11,28 @@ Setup processes and automate activities based on and towards your Knowledge Grap
-- :octicons-terminal-16: [cmemc - Command Line Interface](cmemc-command-line-interface/index.md) +- :octicons-terminal-16: [cmemc - Command Line Interface](cmemc-command-line-interface/index.md) --- cmemc is intended for system administrators and Linked Data experts, who want to automate and control activities on eccenca Corporate Memory remotely. -- :eccenca-artefact-workflow: [Processing data with variable input workflows](processing-data-with-variable-input-workflows/index.md) +- :eccenca-artefact-workflow: [Processing data with variable input workflows](processing-data-with-variable-input-workflows/index.md) --- This tutorial shows how you can create and use data integration workflows to process data coming from outside Corporate Memory (i.e., without registering datasets). -- :material-clock-start: [Scheduling Workflows](scheduling-workflows/index.md) +- :material-clock-start: [Scheduling Workflows](scheduling-workflows/index.md) --- For a time-based execution of a workflow, Corporate Memory provides the Scheduler operator. -- :material-github: [Continuous Integration and Delivery](continuous-integration/index.md) +- :material-github: [Continuous Integration and Delivery](continuous-integration/index.md) --- Setup processes which continuously integrate data artifacts such as vocabularies and shapes with your Corporate Memory instances.
- diff --git a/docs/automate/processing-data-with-variable-input-workflows/index.md b/docs/automate/processing-data-with-variable-input-workflows/index.md index 26e952a1..c3721a52 100644 --- a/docs/automate/processing-data-with-variable-input-workflows/index.md +++ b/docs/automate/processing-data-with-variable-input-workflows/index.md @@ -26,7 +26,7 @@ This allows for solving all kinds of [☆ Automation](../index.md) tasks when yo - by using the [command line interface](../cmemc-command-line-interface/index.md) ``` shell-session - $ cmemc -c my-cmem project import tutorial-varinput.project.zip varinput + cmemc -c my-cmem project import tutorial-varinput.project.zip varinput ``` ## 1 Install the required vocabularies @@ -81,7 +81,7 @@ For this, you need to use the `workflow io` command: ``` shell-session # process one specific feed xml document -$ cmemc workflow io varinput:process-feed -i feed.xml +cmemc workflow io varinput:process-feed -i feed.xml ``` You can easily automate this for a [list of feeds](feeds.txt) like this: diff --git a/docs/automate/scheduling-workflows/index.md b/docs/automate/scheduling-workflows/index.md index 52444713..7c35df02 100644 --- a/docs/automate/scheduling-workflows/index.md +++ b/docs/automate/scheduling-workflows/index.md @@ -30,7 +30,6 @@ Once you are ready with the configurations, click **Create** button. Now, the sc ![Create a Scheduler](22-1-CreateScheduler.gif "Create a Scheduler") - ## Modify, enable or disable a scheduler 1. Navigate to **Build → Projects** section in the workspace. @@ -62,4 +61,3 @@ More common examples: - `PT30M` - every half hour - `PT1H` - every hour - `P1D` - every day - diff --git a/docs/build/active-learning/index.md b/docs/build/active-learning/index.md index 8a501a24..a08d62f9 100644 --- a/docs/build/active-learning/index.md +++ b/docs/build/active-learning/index.md @@ -37,7 +37,7 @@ The examples process below uses the **movies** example project which can be adde ## Creating an automatic link rule -- Choose properties to compare. +- Choose properties to compare. Select from the suggestions or search them by specifying property paths for both entities. ![image](22.2-Suggestion.png){ class="bordered" } @@ -48,15 +48,15 @@ The examples process below uses the **movies** example project which can be adde ## Add property paths for both entities -- Click on the Source path and select a path. +- Click on the Source path and select a path. ![image](22.2-Sourcepath.png){ class="bordered" } -- Click on the Target path and select a corresponding path. +- Click on the Target path and select a corresponding path. ![image](22.2-targetpath.png){ class="bordered" } -- Click on the :eccenca-item-add-artefact: icon to add the path pair to be examined in the learning algorithm. +- Click on the :eccenca-item-add-artefact: icon to add the path pair to be examined in the learning algorithm. ![image](22.2-plusicon.png){ class="bordered" } @@ -66,11 +66,11 @@ The examples process below uses the **movies** example project which can be adde ![image](22.2-stepresult1.png){ class="bordered" } -- Click on :eccenca-item-remove: icon to remove the paths. +- Click on :eccenca-item-remove: icon to remove the paths. ![image](22.2-delete.png){ class="bordered" } -- Click on Start learning. +- Click on Start learning. ![image](22.2-startlearning.png){ class="bordered" } @@ -105,7 +105,7 @@ The examples process below uses the **movies** example project which can be adde ![image](22.2-decline.png){ class="bordered" } -- On the right side of the page click on the 3 dots, then click on show entity’s URI. +- On the right side of the page click on the 3 dots, then click on show entity’s URI. ![image](22.2-uri.png){ class="bordered" } @@ -115,11 +115,11 @@ The examples process below uses the **movies** example project which can be adde ![image](22.2-reflinks.png){ class="bordered" } -- Click on Save based on our input confirm, uncertain and decline the link rule will get generated automatically and the score changes for these entities in the score bar. +- Click on Save based on our input confirm, uncertain and decline the link rule will get generated automatically and the score changes for these entities in the score bar. ![image](22.2-save.png){ class="bordered" } -- Switch on the save best learned rule, then click on save. +- Switch on the save best learned rule, then click on save. ![image](22.2-stepresult2.png){ class="bordered" } diff --git a/docs/build/define-prefixes-namespaces/index.md b/docs/build/define-prefixes-namespaces/index.md index aa20eca1..1ccfdcf2 100644 --- a/docs/build/define-prefixes-namespaces/index.md +++ b/docs/build/define-prefixes-namespaces/index.md @@ -13,8 +13,8 @@ Namespace declarations allow for the abbreviation of IRIs by using a prefixed re For example, after defining a namespace with the values -- **prefix name** = `cohw`, and the -- **namespace IRI** = `https://data.company.org/hardware/` +- **prefix name** = `cohw`, and the +- **namespace IRI** = `https://data.company.org/hardware/` you can use the term `cohw:test` as an abbreviation for the full IRI `https://data.company.org/hardware/test`. @@ -28,8 +28,8 @@ After installing a vocabulary from the [Vocabulary Catalog](../../explore-and-a In order to get the **prefix name** and the **namespace IRI** from the vocabulary graph, the following terms from the [VANN vocabulary](https://vocab.org/vann/) need to be used on the Ontology resource. -- [vann:preferredNamespacePrefix](https://vocab.org/vann/#preferredNamespacePrefix) - to specify the **prefix name** -- [vann:preferredNamespaceUri](https://vocab.org/vann/#preferredNamespaceUri) - to specify the **namespace IRI** +- [vann:preferredNamespacePrefix](https://vocab.org/vann/#preferredNamespacePrefix) - to specify the **prefix name** +- [vann:preferredNamespaceUri](https://vocab.org/vann/#preferredNamespaceUri) - to specify the **namespace IRI** In the Explore area, an Ontology with a correct namespace declaration looks like this: @@ -51,10 +51,10 @@ In addition to the used vocabulary namespace declarations, you may want to add w Such organization use cases include: -- Namespaces per class / resource type: - - **prefix name** = `persons`, **namespace IRI** = `https://example.org/data/persons/` -- Namespaces per data owner or origin: - - **prefix name** = `sales`, **namespace IRI** = `https://example.org/data/sales/` +- Namespaces per class / resource type: + - **prefix name** = `persons`, **namespace IRI** = `https://example.org/data/persons/` +- Namespaces per data owner or origin: + - **prefix name** = `sales`, **namespace IRI** = `https://example.org/data/sales/` Prefixes in Data Integration are defined on a project basis. When creating a new project, a list of well-know prefixes is already declared. @@ -68,8 +68,8 @@ By using the **Edit Prefix Settings** button in this Configuration area, you wil In this dialog, you are able to -- Delete a namespace declaration → **Delete Prefix** -- Add a new namespace declaration → **Add** +- Delete a namespace declaration → **Delete Prefix** +- Add a new namespace declaration → **Add** ## Validating Namespace Declarations diff --git a/docs/build/evaluate-template/index.md b/docs/build/evaluate-template/index.md index fdd76502..c0a22ca9 100644 --- a/docs/build/evaluate-template/index.md +++ b/docs/build/evaluate-template/index.md @@ -30,7 +30,7 @@ The graph dataset is attached to the email as an N-triples file. The following material is used in this tutorial: -- RDF graph containing company information regarding employees, products and services: [company.ttl](company.ttl) +- RDF graph containing company information regarding employees, products and services: [company.ttl](company.ttl) ```Turtle a prod:Hardware ; @@ -294,11 +294,11 @@ The tutorial consists of the following steps, which are described in detail belo 3. Fill in the required details, such as **Label**, your email credentials for sending, and the recipient email address(es). When finished, click **Create**. - - Host: The SMTP host, e.g, mail.myProvider.com - - Port: The SMTP port - - User: The username for the email account - - Password: The password for the email account - - To: The recipient email address(es) + - Host: The SMTP host, e.g, mail.myProvider.com + - Port: The SMTP port + - User: The username for the email account + - Password: The password for the email account + - To: The recipient email address(es)
@@ -320,17 +320,17 @@ The tutorial consists of the following steps, which are described in detail belo Items can be dragged from the list of items on the left side onto the canvas. To connect the outputs and inputs, click and hold the output on the right side of an item and drag it to the input on the left side of another item. - - The **Knowledge Graph dataset** connects to the **Request RDF triples task** and the **SPARQL Select query task**. - - The **Request RDF triples task** connects to the **RDF dataset**. + - The **Knowledge Graph dataset** connects to the **Request RDF triples task** and the **SPARQL Select query task**. + - The **Request RDF triples task** connects to the **RDF dataset**. It requests all triples from the products graph and sends them to the dataset. - - The **RDF dataset** connects to the **Send eMail task**. + - The **RDF dataset** connects to the **Send eMail task**. It holds the NTriples file that will be attached to the email. - - The **SPARQL Select query task** connects to the **Evaluate template task**. + - The **SPARQL Select query task** connects to the **Evaluate template task**. Note that the graph to be queried is specified in the SPARQL query itself with the FROM clause, while the input only triggers its execution. The query results are sent to its output. - - The **Evaluate template task** connects to the **Text dataset**. + - The **Evaluate template task** connects to the **Text dataset**. It receives the SPARQL query results and sends the evaluated Jinja template to its output. - - The **Text dataset** connects to the **Transform**. + - The **Text dataset** connects to the **Transform**. It holds the text file with the evaluated Jinja template and acts as input for the Transform. ![Workflow 1](workflow-1.png){ class="bordered" } @@ -340,11 +340,11 @@ The tutorial consists of the following steps, which are described in detail belo The **Evaluate template** operator can also be connected directly to the **Transform**. In this case, skip [§6](#6-create-a-text-dataset) and enter *output* instead of *text* for the **Value path** of the value mapping in the **Transform** (see [§7.6](#7-create-a-transform)). -1. Click on three dots of the **Send eMail** task, select **Config** and tick the check box to enable the config port. +5. Click on three dots of the **Send eMail** task, select **Config** and tick the check box to enable the config port. ![Set Config Port](config-port.png){ class="bordered" width="55%" } -2. Connect the output of the **Transform** to the config port located on the top of the **Send eMail** task. +6. Connect the output of the **Transform** to the config port located on the top of the **Send eMail** task. When finished, click **Save**. The complete workflow now looks as shown below. diff --git a/docs/build/extracting-data-from-a-web-api/index.md b/docs/build/extracting-data-from-a-web-api/index.md index e0b47b4a..975525b7 100644 --- a/docs/build/extracting-data-from-a-web-api/index.md +++ b/docs/build/extracting-data-from-a-web-api/index.md @@ -18,13 +18,13 @@ The tutorial is based on the [GitHub API (v3)](https://developer.github.com/v3/) - by using the [command line interface](../../automate/cmemc-command-line-interface/index.md) ``` shell-session - $ cmemc -c my-cmem project import tutorial-webapi.project.zip web-api + cmemc -c my-cmem project import tutorial-webapi.project.zip web-api ``` In order to get familiar with the API, simply fetch an example response with this command: ``` shell-session -$ curl https://api.github.com/orgs/vocol/repos +curl https://api.github.com/orgs/vocol/repos ``` The HTTP Get request retrieves all repositories of a GitHub organization named vocol. @@ -65,7 +65,6 @@ The JSON response includes the data for all repositories (**mobivoc**, **vocol** … --> - ## 1 Register a Web API @@ -159,4 +158,3 @@ To build a workflow that combines all the elements we previously built, we now d 5. Validate the result by clicking on the **Workflow** **Report** tab and see the result of your execution. In this example, 15x repositories were found from the GitHub API request. ![Workflow execution report](extract-from-api-wf-report.png) - diff --git a/docs/build/index.md b/docs/build/index.md index a6e4133f..a7ab41c4 100644 --- a/docs/build/index.md +++ b/docs/build/index.md @@ -15,7 +15,7 @@ The Build stage turns your source data—across files, databases, APIs, and stre
-- :eccenca-application-dataintegration: Foundations: Introduction and Best Practices +- :eccenca-application-dataintegration: Foundations: Introduction and Best Practices --- @@ -25,7 +25,7 @@ The Build stage turns your source data—across files, databases, APIs, and stre - [Define Prefixes / Namespaces](define-prefixes-namespaces/index.md) --- Namespace declarations allow for abbreviation of IRIs by using a prefixed name instead of an IRI, in particular when writing SPARQL queries or Turtle. - [Spark](spark/index.md) --- Explainer of Apache Spark and its integration within the BUILD platform. -- :material-list-status: Tutorials +- :material-list-status: Tutorials --- @@ -39,14 +39,14 @@ The Build stage turns your source data—across files, databases, APIs, and stre - [Evaluate Jinja Template and Send an Email Message](evaluate-template/index.md) --- Template and send an email after a workflow execution. - [Link Intrusion Detection Systems to Open-Source INTelligence](tutorial-how-to-link-ids-to-osint/index.md) --- Link IDS data to OSINT sources. -- :fontawesome-regular-snowflake: Patterns +- :fontawesome-regular-snowflake: Patterns --- - [Reconfigure Workflow Tasks](workflow-reconfiguration/index.md) --- During its execution, new parameters can be loaded from any source, which overwrites originally set parameters. - [Project and Global Variables](variables/index.md) --- Define and reuse variables across tasks and projects. -- :material-book-open-variant-outline: Reference +- :material-book-open-variant-outline: Reference --- @@ -55,4 +55,3 @@ The Build stage turns your source data—across files, databases, APIs, and stre - [Task and Operator Reference](reference/index.md) --- Reference documentation for tasks and operators in the Build workspace.
- diff --git a/docs/build/integrations/index.md b/docs/build/integrations/index.md index da6ae283..027e876f 100644 --- a/docs/build/integrations/index.md +++ b/docs/build/integrations/index.md @@ -7,14 +7,16 @@ tags: - Build - Reference --- + # Integrations + The following services and applications can be easily integrated in Corporate Memory workflows:
-- :simple-anthropic:{ .lg .middle } Anthropic / Claude +- :simple-anthropic:{ .lg .middle } Anthropic / Claude --- @@ -24,14 +26,14 @@ to interact with any (LLMs). -- :other-apacheavro:{ .lg .middle } Avro +- :other-apacheavro:{ .lg .middle } Avro --- Use the [Avro](../../build/reference/dataset/avro.md) dataset to read and write files in the [Avro format](https://avro.apache.org/). -- :material-microsoft-azure:{ .lg .middle } Azure AI Foundry +- :material-microsoft-azure:{ .lg .middle } Azure AI Foundry --- @@ -39,7 +41,7 @@ to interact with any to interact with any [Azure AI Foundry provided Large Language Models](https://ai.azure.com/catalog) (LLMs). -- :fontawesome-solid-file-csv:{ .lg .middle } CSV +- :fontawesome-solid-file-csv:{ .lg .middle } CSV --- @@ -47,85 +49,87 @@ to interact with any [Azure AI Foundry provided Large Language Models](https://a (read and write) with the [CSV Dataset](../../build/reference/dataset/csv.md). -- :simple-databricks:{ .lg .middle } Databricks +- :simple-databricks:{ .lg .middle } Databricks --- [Databricks](http://databricks.com/) can be accessed with the [Remote SQL endpoint](../../build/reference/dataset/Jdbc.md) dataset and a [Custom JDBC driver](https://github.com/databricks/databricks-jdbc). Please have a look at + [Setup and use of JDBC Drivers](../../deploy-and-configure/configuration/dataintegration/jdbc/index.md). -- :material-email-outline:{ .lg .middle } eMail / SMTP +- :material-email-outline:{ .lg .middle } eMail / SMTP --- Send plain text or HTML formatted [eMail messages](../../build/reference/customtask/SendEMail.md) using an SMTP server. -- :material-file-excel:{ .lg .middle } Excel +- :material-file-excel:{ .lg .middle } Excel --- Use the [Excel](../../build/reference/dataset/excel.md) task to read and write to Excel workbooks in the Open XML format (XLSX). -- :material-google-drive:{ .lg .middle } Google Drive +- :material-google-drive:{ .lg .middle } Google Drive --- Use the [Excel (Google Drive)](../../build/reference/dataset/googlespreadsheet.md) to read and write to Excel workbooks in Google Drive. -- :other-graphdb:{ .lg .middle } GraphDB +- :other-graphdb:{ .lg .middle } GraphDB --- Load and write Knowledge Graphs to an external GraphDB store by using the [SPARQL endpoint](../../build/reference/dataset/sparqlEndpoint.md) dataset. Query data from GraphDB by using the SPARQL + [Construct](../../build/reference/customtask/sparqlCopyOperator.md), [Select](../../build/reference/customtask/sparqlSelectOperator.md) and [Update](../../build/reference/customtask/sparqlUpdateOperator.md) tasks. GraphDB can be used as the integrated Quad Store as well. -- :simple-graphql:{ .lg .middle } GraphQL +- :simple-graphql:{ .lg .middle } GraphQL --- You can execute a [GraphQL query](../../build/reference/customtask/cmem_plugin_graphql-workflow-graphql-GraphQLPlugin.md) and process the result in a workflow. -- :simple-apachehive:{ .lg .middle } Hive +- :simple-apachehive:{ .lg .middle } Hive --- Read from or write to an embedded Apache [Hive database](../../build/reference/dataset/Hive.md) endpoint. -- :simple-jira:{ .lg .middle } Jira +- :simple-jira:{ .lg .middle } Jira --- Execute a [JQL query](../../build/reference/customtask/cmem_plugin_jira-JqlQuery.md) on a Jira instance to fetch and integrate issue data. -- :material-code-json:{ .lg .middle } JSON +- :material-code-json:{ .lg .middle } JSON --- Use the [JSON](../../build/reference/dataset/json.md) dataset to read and write JSON files (JavaScript Object Notation). -- :material-code-json:{ .lg .middle } JSON Lines +- :material-code-json:{ .lg .middle } JSON Lines --- Use the [JSON](../../build/reference/dataset/json.md) dataset to read and write files in the [JSON Lines](https://jsonlines.org/) text file format. -- :simple-apachekafka:{ .lg .middle } Kafka +- :simple-apachekafka:{ .lg .middle } Kafka --- @@ -133,21 +137,21 @@ GraphDB can be used as the integrated Quad Store as well. [receive messages](../../build/reference/customtask/cmem_plugin_kafka-ReceiveMessages.md) to and from a Kafka topic. -- :simple-kubernetes:{ .lg .middle } Kubernetes +- :simple-kubernetes:{ .lg .middle } Kubernetes --- You can [Execute a command in a kubernetes pod](../../build/reference/customtask/cmem_plugin_kubernetes-Execute.md) and captures its output to process it. -- :simple-mariadb:{ .lg .middle } MariaDB +- :simple-mariadb:{ .lg .middle } MariaDB --- MariaDB can be accessed with the [Remote SQL endpoint](../../build/reference/dataset/Jdbc.md) dataset (JDBC driver included). -- :simple-mattermost:{ .lg .middle } Mattermost +- :simple-mattermost:{ .lg .middle } Mattermost --- @@ -155,40 +159,41 @@ GraphDB can be used as the integrated Quad Store as well. the [Send Mattermost messages](../../build/reference/customtask/cmem_plugin_mattermost.md) task. -- :material-microsoft:{ .lg .middle } Microsoft SQL +- :material-microsoft:{ .lg .middle } Microsoft SQL --- The Microsoft SQL Server can be accessed with the [Remote SQL endpoint](../../build/reference/dataset/Jdbc.md) dataset (JDBC driver included). -- :simple-mysql:{ .lg .middle } MySQL +- :simple-mysql:{ .lg .middle } MySQL --- MySQL can be accessed with the [Remote SQL endpoint](../../build/reference/dataset/Jdbc.md) dataset (JDBC driver included). -- :simple-neo4j:{ .lg .middle } Neo4J +- :simple-neo4j:{ .lg .middle } Neo4J --- Use the [Neo4j](../../build/reference/dataset/neo4j.md) dataset for reading and writing [Neo4j graphs](https://neo4j.com/). -- :other-neptune:{ .lg .middle } Neptune +- :other-neptune:{ .lg .middle } Neptune --- Load and write Knowledge Graphs to Amazon Neptune by using the [SPARQL endpoint](../../build/reference/dataset/sparqlEndpoint.md) dataset. Query data from Amazon Neptune by using the SPARQL + [Construct](../../build/reference/customtask/sparqlCopyOperator.md), [Select](../../build/reference/customtask/sparqlSelectOperator.md) and [Update](../../build/reference/customtask/sparqlUpdateOperator.md) tasks. Amazon Neptune can be used as the integrated Quad Store as well (beta). -- :simple-nextcloud:{ .lg .middle } Nextcloud +- :simple-nextcloud:{ .lg .middle } Nextcloud --- @@ -196,14 +201,14 @@ Amazon Neptune can be used as the integrated Quad Store as well (beta). them or [upload files](../../build/reference/customtask/cmem_plugin_nextcloud-Upload.md) you created with Corporate Memory. -- :material-microsoft-office:{ .lg .middle } Office 365 +- :material-microsoft-office:{ .lg .middle } Office 365 --- Use the [Excel (OneDrive, Office365)](../../build/reference/dataset/office365preadsheet.md) to read and write to Excel workbooks in Office 365. -- :simple-ollama:{ .lg .middle } Ollama +- :simple-ollama:{ .lg .middle } Ollama --- @@ -211,7 +216,7 @@ them or [upload files](../../build/reference/customtask/cmem_plugin_nextcloud-Up to interact with Ollama provided Large Language Models (LLMs). -- :simple-openai:{ .lg .middle } OpenAI +- :simple-openai:{ .lg .middle } OpenAI --- @@ -219,7 +224,7 @@ to interact with Ollama provided Large Language Models (LLMs). to interact with any [OpenAI provided Large Language Models](https://platform.openai.com/docs/models) (LLMs). -- :octicons-ai-model-24:{ .lg .middle } OpenRouter +- :octicons-ai-model-24:{ .lg .middle } OpenRouter --- @@ -227,21 +232,21 @@ to interact with any [OpenAI provided Large Language Models](https://platform.op to interact with any [OpenRouter provided Large Language Models](https://openrouter.ai/models) (LLMs). -- :other-apacheorc:{ .lg .middle } ORC +- :other-apacheorc:{ .lg .middle } ORC --- Use the [ORC](../../build/reference/dataset/orc.md) dataset to read and write files in the [ORC](https://orc.apache.org/) format. -- :simple-apacheparquet:{ .lg .middle } Parquet +- :simple-apacheparquet:{ .lg .middle } Parquet --- Use the [Parquet](../../build/reference/dataset/parquet.md) dataset to read and write files in the [Parquet](https://parquet.apache.org/) format. -- :black_large_square:{ .lg .middle } pgvector +- :black_large_square:{ .lg .middle } pgvector --- @@ -249,14 +254,14 @@ to interact with any [OpenRouter provided Large Language Models](https://openrou using the [Search Vector Embeddings](../../build/reference/customtask/cmem_plugin_pgvector-Search.md). -- :simple-postgresql:{ .lg .middle } PostgreSQL +- :simple-postgresql:{ .lg .middle } PostgreSQL --- PostgreSQL can be accessed with the [Remote SQL endpoint](../../build/reference/dataset/Jdbc.md) dataset (JDBC driver included). -- :other-powerbi:{ .lg .middle } PowerBI +- :other-powerbi:{ .lg .middle } PowerBI --- @@ -264,19 +269,20 @@ using the [Search Vector Embeddings](../../build/reference/customtask/cmem_plugi [Corporate Memory Power-BI-Connector](../../consume/consuming-graphs-in-power-bi/index.md). -- :other-qlever:{ .lg .middle } Qlever +- :other-qlever:{ .lg .middle } Qlever --- Load and write Knowledge Graphs to an external Qlever store by using the [SPARQL endpoint](../../build/reference/dataset/sparqlEndpoint.md) dataset. Query data from Qlever by using the SPARQL + [Construct](../../build/reference/customtask/sparqlCopyOperator.md), [Select](../../build/reference/customtask/sparqlSelectOperator.md) and [Update](../../build/reference/customtask/sparqlUpdateOperator.md) tasks. Qlever can be used as the integrated Quad Store as well (beta). -- :simple-semanticweb:{ .lg .middle } RDF +- :simple-semanticweb:{ .lg .middle } RDF --- @@ -286,7 +292,7 @@ Qlever can be used as the integrated Quad Store as well (beta). [RDF/JSON](https://www.w3.org/TR/rdf-json/)). -- :other-redash:{ .lg .middle } Redash +- :other-redash:{ .lg .middle } Redash --- @@ -294,14 +300,14 @@ Qlever can be used as the integrated Quad Store as well (beta). [Corporate Memory Redash-Connector](../../consume/consuming-graphs-with-redash/index.md). -- :material-application-braces-outline:{ .lg .middle } REST +- :material-application-braces-outline:{ .lg .middle } REST --- Execute REST requests using [Execute REST requests](../../build/reference/customtask/eccencaRestOperator.md). -- :fontawesome-brands-salesforce:{ .lg .middle } Salesforce +- :fontawesome-brands-salesforce:{ .lg .middle } Salesforce --- @@ -309,72 +315,76 @@ Qlever can be used as the integrated Quad Store as well (beta). execute a [SOQL query (Salesforce)](../../build/reference/customtask/cmem_plugin_salesforce-SoqlQuery.md). -- :simple-snowflake:{ .lg .middle } Snowflake +- :simple-snowflake:{ .lg .middle } Snowflake --- Snowflake can be accessed with the [Snowflake SQL endpoint](../../build/reference/dataset/SnowflakeJdbc.md) dataset (JDBC driver included). -- :simple-apachespark:{ .lg .middle } Spark +- :simple-apachespark:{ .lg .middle } Spark --- Apply a [Spark](https://spark.apache.org/) function to a specified field using [Execute Spark function](../../build/reference/customtask/SparkFunction.md). -- :simple-sqlite:{ .lg .middle } SQLite +- :simple-sqlite:{ .lg .middle } SQLite --- SQLite can be accessed with the [Remote SQL endpoint](../../build/reference/dataset/Jdbc.md) dataset and a [Custom JDBC driver](https://central.sonatype.com/artifact/org.xerial/sqlite-jdbc). Please have a look at + [Setup and use of JDBC Drivers](../../deploy-and-configure/configuration/dataintegration/jdbc/index.md). -- :material-ssh:{ .lg .middle } SSH +- :material-ssh:{ .lg .middle } SSH --- Interact with SSH servers to [Download SSH files](../../build/reference/customtask/cmem_plugin_ssh-Download.md) or [Execute commands via SSH](../../build/reference/customtask/cmem_plugin_ssh-Execute.md). -- :other-tentris:{ .lg .middle } Tentris +- :other-tentris:{ .lg .middle } Tentris --- Load and write Knowledge Graphs to an external Tentris store by using the [SPARQL endpoint](../../build/reference/dataset/sparqlEndpoint.md) dataset. Query data from Tentris by using the SPARQL + [Construct](../../build/reference/customtask/sparqlCopyOperator.md), [Select](../../build/reference/customtask/sparqlSelectOperator.md) and [Update](../../build/reference/customtask/sparqlUpdateOperator.md) tasks. Tentris can be used as the integrated Quad Store as well (beta). -- :simple-trino:{ .lg .middle } Trino +- :simple-trino:{ .lg .middle } Trino --- [Trino](https://github.com/trinodb/trino) can be access with the [Remote SQL endpoint](../../build/reference/dataset/Jdbc.md) dataset and a [Custom JDBC driver](https://trino.io/docs/current/client/jdbc.html). Please have a look at + [Setup and use of JDBC Drivers](../../deploy-and-configure/configuration/dataintegration/jdbc/index.md). -- :black_large_square:{ .lg .middle } Virtuoso +- :black_large_square:{ .lg .middle } Virtuoso --- Load and write Knowledge Graphs to an external Openlink Virtuoso store by using the [SPARQL endpoint](../../build/reference/dataset/sparqlEndpoint.md) dataset. Query data from Virtuoso by using the SPARQL + [Construct](../../build/reference/customtask/sparqlCopyOperator.md), [Select](../../build/reference/customtask/sparqlSelectOperator.md) and [Update](../../build/reference/customtask/sparqlUpdateOperator.md) tasks. Virtuoso can be used as the integrated Quad Store as well (beta). -- :material-xml:{ .lg .middle } XML +- :material-xml:{ .lg .middle } XML --- @@ -382,14 +392,14 @@ Virtuoso can be used as the integrated Quad Store as well (beta). [Parse XML](../../build/reference/customtask/XmlParserOperator.md) from external services. -- :simple-yaml:{ .lg .middle } YAML +- :simple-yaml:{ .lg .middle } YAML --- Load and integrate data from YAML files with the [Parse YAML](../../build/reference/customtask/cmem_plugin_yaml-parse.md) task. -- :material-code-json:{ .lg .middle } Zipped JSON +- :material-code-json:{ .lg .middle } Zipped JSON --- @@ -398,4 +408,4 @@ Virtuoso can be used as the integrated Quad Store as well (beta). -
\ No newline at end of file + diff --git a/docs/build/kafka-consumer/index.md b/docs/build/kafka-consumer/index.md index 3d2f7fb9..907a6abe 100644 --- a/docs/build/kafka-consumer/index.md +++ b/docs/build/kafka-consumer/index.md @@ -53,18 +53,18 @@ In Create new item window, select Kafka Consumer (Receive Messages) and click Ad Configure the Kafka Consumer according to the topic that shall be consumed: -- **Bootstrap Server** - URL of the Kafka broker including the port number (commonly port ´9092) -- **Security Protocol** - Security mechanism used for authentication -- **Topic** - Name / ID of the topic where messages are published -- **Advanced Section** - - **Messages Dataset** - A dataset (XML/JSON) where messages can be written to. Leave this field empty to output the messages as entities (see below). - - **SASL** authentication settings as provided by your Kafka broker - - **Auto Offset Reset** - Consumption starts either at the earliest offset or the latest offset. - - **Consumer Group Name** - Consumer groups can be used to distribute the load of messages (partitions) between multiple consumers of the same group (c.f. [Kafka Concepts](https://docs.confluent.io/platform/current/clients/consumer.html#concepts)). - - **Client Id** - An optional identifier of the client which is communicated to the server. When this field is empty, the plugin defaults to `DNS:PROJECT_ID:TASK_ID`. - - **Local Consumer Queue Size** - Maximum total message size in kilobytes that the consumer can buffer for a specific partition. The consumer will stop fetching from the partition if it hits this limit. This helps prevent consumers from running out of memory. - - **Message Limit** - The maximum number of messages to fetch and process in each run. If `0` or less, all messages will be fetched. - - **Disable Commit** Setting this to `true` will disable committing messages after retrival. This means you will receive the same messages on the next execution (for testing, development, or debugging). +- **Bootstrap Server** - URL of the Kafka broker including the port number (commonly port ´9092) +- **Security Protocol** - Security mechanism used for authentication +- **Topic** - Name / ID of the topic where messages are published +- **Advanced Section** + - **Messages Dataset** - A dataset (XML/JSON) where messages can be written to. Leave this field empty to output the messages as entities (see below). + - **SASL** authentication settings as provided by your Kafka broker + - **Auto Offset Reset** - Consumption starts either at the earliest offset or the latest offset. + - **Consumer Group Name** - Consumer groups can be used to distribute the load of messages (partitions) between multiple consumers of the same group (c.f. [Kafka Concepts](https://docs.confluent.io/platform/current/clients/consumer.html#concepts)). + - **Client Id** - An optional identifier of the client which is communicated to the server. When this field is empty, the plugin defaults to `DNS:PROJECT_ID:TASK_ID`. + - **Local Consumer Queue Size** - Maximum total message size in kilobytes that the consumer can buffer for a specific partition. The consumer will stop fetching from the partition if it hits this limit. This helps prevent consumers from running out of memory. + - **Message Limit** - The maximum number of messages to fetch and process in each run. If `0` or less, all messages will be fetched. + - **Disable Commit** Setting this to `true` will disable committing messages after retrival. This means you will receive the same messages on the next execution (for testing, development, or debugging). ![Configuration options](configure-kafka-consumer.png){ class="bordered" } @@ -86,11 +86,11 @@ To execute the Kafka Consumer it needs to be placed inside a Workflow. The messa In the "message streaming mode" (**Messages Dataset** is not set) the received messages will be generated as entities and forwarded to the subsequent operator in the workflow. This mode is not limited to any message format. The generated message entities will have the following flat schema: -- **key** — the optional key of the message, -- **content** — the message itself as plain text, -- **offset** — the given offset of the message in the topic, -- **ts-production** — the timestamp when the message was written to the topic, -- **ts-consumption** — the timestamp when the message was consumed from the topic. +- **key** — the optional key of the message, +- **content** — the message itself as plain text, +- **offset** — the given offset of the message in the topic, +- **ts-production** — the timestamp when the message was written to the topic, +- **ts-consumption** — the timestamp when the message was consumed from the topic. Connect the output of Kafka Consumer inside a Workflow to a tabular dataset (e.g. a [CSV Dataset](../reference/dataset/csv.md)) or directly to a transformation task. diff --git a/docs/build/lift-data-from-json-and-xml-sources/index.md b/docs/build/lift-data-from-json-and-xml-sources/index.md index eed44537..ea2d30af 100644 --- a/docs/build/lift-data-from-json-and-xml-sources/index.md +++ b/docs/build/lift-data-from-json-and-xml-sources/index.md @@ -30,11 +30,11 @@ The documentation consists of the following steps, which are described in detail The following material is used in this tutorial: -- Sample vocabulary describing the data in the JSON and XML files: [products_vocabulary.nt](products_vocabulary.nt) +- Sample vocabulary describing the data in the JSON and XML files: [products_vocabulary.nt](products_vocabulary.nt) ![Visualization of the "Products Vocabulary".](products-vocab-xml+json.png){ class="bordered" } -- Sample JSON file: [services.json](services.json) +- Sample JSON file: [services.json](services.json) ```json [ @@ -56,7 +56,7 @@ The following material is used in this tutorial: ] ``` -- Sample XML file: [orgmap.xml](orgmap.xml) +- Sample XML file: [orgmap.xml](orgmap.xml) ```xml @@ -119,9 +119,9 @@ The vocabulary contains the classes and properties needed to map the source data 3. Define a **Name**, a **Graph URI** and a **Description** of the vocabulary. _In this example we will use:_ - - Name: _**Product Vocabulary**_ - - Graph URI: _****_ - - Description: _**Example vocabulary modeled to describe relations between products and services.**_ + - Name: _**Product Vocabulary**_ + - Graph URI: _****_ + - Description: _**Example vocabulary modeled to describe relations between products and services.**_ ![Dialog to register a new vocabulary.](dialog-register-new-vocabulary.png){ class="bordered" width="50%" } @@ -339,8 +339,8 @@ Click **Transform evaluation** to evaluate the transformed entities. 2. Press the ![Button play](button-play-xml+json.png) button and validate the results. In this example, 9x Service entities were created in our Knowledge Graph based on the mapping. 3. You can click **Knowledge Graphs** under **EXPLORE** to (re-)view of the created Knowledge Graphs 4. Enter the following URIs in the Enter search term for JSON and XML respectively. - - JSON / Service: _****_ - - XML / Department: _****_ + - JSON / Service: _****_ + - XML / Department: _****_ === "JSON" diff --git a/docs/build/lift-data-from-tabular-data-such-as-csv-xslx-or-database-tables/index.md b/docs/build/lift-data-from-tabular-data-such-as-csv-xslx-or-database-tables/index.md index e6788d6b..e7fe2d93 100644 --- a/docs/build/lift-data-from-tabular-data-such-as-csv-xslx-or-database-tables/index.md +++ b/docs/build/lift-data-from-tabular-data-such-as-csv-xslx-or-database-tables/index.md @@ -19,7 +19,7 @@ This beginner-level tutorial shows how you can build a Knowledge Graph based on - by using the [command line interface](../../automate/cmemc-command-line-interface/index.md) ``` shell-session - $ cmemc -c my-cmem project import tutorial-csv.project.zip tutorial-csv + cmemc -c my-cmem project import tutorial-csv.project.zip tutorial-csv ``` This step is optional and makes some of the following steps of the tutorial superfluous. @@ -33,16 +33,15 @@ The documentation consists of the following steps, which are described in detail 5. Evaluate a Transformation 6. Build the Knowledge Graph - ## Sample Material The following material is used in this tutorial, you should download the files and have them at hand throughout the tutorial: -- Sample vocabulary which describes the data in the CSV files: [products_vocabulary.nt](products_vocabulary.nt) +- Sample vocabulary which describes the data in the CSV files: [products_vocabulary.nt](products_vocabulary.nt) ![](products-vocab.png){ class="bordered" } -- Sample CSV file: [services.csv](services.csv) +- Sample CSV file: [services.csv](services.csv) !!! info @@ -52,7 +51,7 @@ The following material is used in this tutorial, you should download the files a | I241-8776317 | Component Confabulation | Z249-1364492, L557-1467804, C721-7900144, ... | Corinna.Ludwig@company.org | 1082,00 EUR | | … | … | … | … | … | -- Sample Excel file: [products.xlsx](products.xlsx) +- Sample Excel file: [products.xlsx](products.xlsx) !!! info @@ -87,11 +86,10 @@ The vocabulary contains the classes and properties needed to map the data into t ![Register new Vocabulary](register-new-vocab.png){ class="bordered" width="50%" } - === "cmemc" ``` shell-session - $ cmemc vocabulary import products_vocabulary.nt + cmemc vocabulary import products_vocabulary.nt ``` --- @@ -102,7 +100,7 @@ The vocabulary contains the classes and properties needed to map the data into t ![](menu-build-projects.png){ class="bordered" width="50%" } -2. Click **Create :octicons-plus-circle-24:** at the top right of the page.  +2. Click **Create :octicons-plus-circle-24:** at the top right of the page. 3. In the **Create new item** window, select **Project** and click **Add**. The Create new item of type Project window appears.   @@ -110,7 +108,6 @@ The vocabulary contains the classes and properties needed to map the data into t 5. Click **Create**. Your project is created. - --- === "Workflow view" @@ -161,7 +158,7 @@ The vocabulary contains the classes and properties needed to map the data into t The general form of the JDBC connection string is: - ``` + ```text jdbc:://:/ ``` @@ -198,7 +195,6 @@ The transformation defines how an input dataset (e.g. CSV) will be transformed i ![](transformation-label.png){ class="bordered" width="50%" } - 3. Scroll down to **Target vocabularies** and choose **Products vocabulary**. ![](select-vocabulary.png){ class="bordered" width="50%" } @@ -219,13 +215,13 @@ The transformation defines how an input dataset (e.g. CSV) will be transformed i 4. Define the **Target entity type** from the vocabulary, the **URI pattern** and a **label** for the mapping. _In this example we will use:_ - - Target entity type: _**Service**_ - - URI pattern: + - Target entity type: _**Service**_ + - URI pattern: - - Click **Create custom pattern** - - Insert `http://ld.company.org/prod-inst/{ServiceID}`, where `http://ld.company.org/prod-inst/` is a common prefix for the instances in this use case, and `{ServiceID}` is a placeholder that will resolve to the column of that name. + - Click **Create custom pattern** + - Insert `http://ld.company.org/prod-inst/{ServiceID}`, where `http://ld.company.org/prod-inst/` is a common prefix for the instances in this use case, and `{ServiceID}` is a placeholder that will resolve to the column of that name. - - An optional Label: `Service` + - An optional Label: `Service` ![](services-mapping-class.png){ class="bordered" width="50%" } @@ -237,26 +233,26 @@ _Example RDF triple in our Knowledge Graph based on the mapping definition:_ ``` -6. Evaluate your mapping by clicking the Expand :material-greater-than: button in the **Examples of target data** property to see at most three generated base URIs. +1. Evaluate your mapping by clicking the Expand :material-greater-than: button in the **Examples of target data** property to see at most three generated base URIs. ![](mapping-inline-preview.png){ class="bordered" width="50%" } We have now created the Service entities in the Knowledge Graph. As a next step, we will add the name of the Service entity. -7. Press the circular **Blue + button** on the lower right and select **Add value mapping**. +2. Press the circular **Blue + button** on the lower right and select **Add value mapping**. ![](services-mapping-add-rule.png){ class="bordered" width="50%" } -8. Define the **Target property**, the **Data type**, the **Value path** (column name) and a **Label** for your value mapping. _In this example we will use:_ +3. Define the **Target property**, the **Data type**, the **Value path** (column name) and a **Label** for your value mapping. _In this example we will use:_ - - Target Property: `name` - - Data type: _**String**_ - - Value path: `ServiceName` (which corresponds to the column of that name) - - An optional Label: `service name` + - Target Property: `name` + - Data type: _**String**_ + - Value path: `ServiceName` (which corresponds to the column of that name) + - An optional Label: `service name` ![](services-mapping-rule-edit.png){ class="bordered" width="50%" } -9. Click **Save**. +4. Click **Save**. --- @@ -266,7 +262,6 @@ Go the **Transform evaluation** tab of your transformation to view a list of gen ![](mapping-evaluation.png){ class="bordered" width="50%" } - --- ## 6 Build the Knowledge Graph @@ -279,8 +274,8 @@ Go the **Transform evaluation** tab of your transformation to view a list of gen 3. Define a **Label** for the Knowledge Graph and provide a **graph** uri. Leave all the other parameters at the default values. _In this example we will use:_ - - Label: `Service Knowledge Graph` - - Graph: `http://ld.company.org/prod-instances/` + - Label: `Service Knowledge Graph` + - Graph: `http://ld.company.org/prod-instances/` ![](knowledge-graph.png){ class="bordered" width="50%" } @@ -292,7 +287,6 @@ Go the **Transform evaluation** tab of your transformation to view a list of gen ![](mapping-execution-result.png){ class="bordered" width="50%" } - 7. Click Knowledge Graph under **Explore** in the navigation on the left side of the page. ![](explore-knowledge-graph.png){ class="bordered" width="50%" } @@ -308,4 +302,3 @@ Go the **Transform evaluation** tab of your transformation to view a list of gen 10. Finally you can use the Explore **Knowledge Graphs** module to (re-)view of the created Knowledge Graph: `http://ld.company.org/prod-instances/` ![](kg-result.png){ class="bordered" width="50%" } - diff --git a/docs/build/loading-jdbc-datasets-incrementally/index.md b/docs/build/loading-jdbc-datasets-incrementally/index.md index 29bfc3db..3461908b 100644 --- a/docs/build/loading-jdbc-datasets-incrementally/index.md +++ b/docs/build/loading-jdbc-datasets-incrementally/index.md @@ -46,15 +46,15 @@ To extract data from a relational database, you need to first register a **JDBC ![Create JDBC Dataset](create-dataset-JDBC.png){ class="bordered" } 5. Provide the required configuration details for the JDBC endpoint: - - **Label**: Provide a table name. - - **Description:** Optionally describe your table. - - **JDBC Driver Connection URL:** Provide the JDBC connection. In this tutorial we use a MySQL database. The database server is named _mysql_ and the database is named _serviceDB_. - - **Table:** Provide the name of the table in the database. - - **Source query**: Provide a default source query. In this tutorial, the source query will be modified later as the OFFSET changes. - - **Limit:** Provide a LIMIT for the SQL query. In this tutorial, we choose 5 for demonstrating the functionality. You may select any value which works for your use case. - - **Query strategy**: Select: _Execute the given source query. No paging or virtual Query._ In this tutorial, this needs to be changed so that when this JDBC endpoint is being used, Corporate Memory will always check for the _Source Query_ that was provided earlier. - - **User**: Provide the user name which is allowed to access the database. - - **Password**: Provide the user password that is allowed to access the database. + - **Label**: Provide a table name. + - **Description:** Optionally describe your table. + - **JDBC Driver Connection URL:** Provide the JDBC connection. In this tutorial we use a MySQL database. The database server is named _mysql_ and the database is named _serviceDB_. + - **Table:** Provide the name of the table in the database. + - **Source query**: Provide a default source query. In this tutorial, the source query will be modified later as the OFFSET changes. + - **Limit:** Provide a LIMIT for the SQL query. In this tutorial, we choose 5 for demonstrating the functionality. You may select any value which works for your use case. + - **Query strategy**: Select: _Execute the given source query. No paging or virtual Query._ In this tutorial, this needs to be changed so that when this JDBC endpoint is being used, Corporate Memory will always check for the _Source Query_ that was provided earlier. + - **User**: Provide the user name which is allowed to access the database. + - **Password**: Provide the user password that is allowed to access the database. ![Configuration of a JDBC dataset](configure-JDBC-1.png){ class="bordered" } @@ -72,7 +72,7 @@ To incrementally extract data in Corporate Memory, we need to store the informat 4. Select the previously created JDBC endpoint (in our example: "Services Table (JDBC)" 5. Press the **Turtle** tab inside your JDBC endpoint view (right) -In our example, the JDBC Endpoint IRI looks like this: __IncrementalJDBCdatasetload/8d0e4895-1d45-442f-8fd8-b1459ec3dbde_ServicesTableJDBC_ +In our example, the JDBC Endpoint IRI looks like this: `` See screenshot below for example: @@ -86,7 +86,14 @@ The following three RDF triples hold the (minimal) necessary information we need 2. The second triple defines a label for the Graph. 3. The third triple defines the <...**lastOffset**> property we need for this tutorial. As a default, we set it to 0 to start with the first row in the table. -**services_metadata_graph** +For your project: + +1. adjust the CMEM DI Project IRI and +2. the JDBC endpoint IRI. + +**Import the Graph** in the Exploration tab → Graph (menu) → Add new Graph → Provide Graph IRI + Select file. + +`services_metadata_graph.nt`: ```nt @@ -100,14 +107,7 @@ The following three RDF triples hold the (minimal) necessary information we need "0" . # set the initial offset to zero to start with the first row in the table ``` -For your project, please: - -1. adjust the CMEM DI Project IRI and -2. the JDBC endpoint IRI. - -**Import the Graph** in the Exploration tab → Graph (menu) → Add new Graph → Provide Graph IRI + Select file - -In our example, we used the following Graph IRI for the Metadata Graph: __ +In our example, we used the following Graph IRI for the Metadata Graph: `` ## 3 Create a Transformation to dynamically compose a SQL Query diff --git a/docs/build/mapping-creator/index.md b/docs/build/mapping-creator/index.md index 0f019324..a15409dc 100644 --- a/docs/build/mapping-creator/index.md +++ b/docs/build/mapping-creator/index.md @@ -34,9 +34,9 @@ Using visual tools, drag-and-drop, and suggestions, you can create mappings betw The Mapping Creator consists of three parts: -- Source schema shown on the left side -- Target Schema shown on the right side -- Mappings between elements in the source schema and in the target schema +- Source schema shown on the left side +- Target Schema shown on the right side +- Mappings between elements in the source schema and in the target schema You can move, connect or disconnect, and inspect each element visually. @@ -65,8 +65,8 @@ To complete a mapping, properties need to be added to complete your desired targ There are two options to add properties: -- during class selection -- from vocabularies +- during class selection +- from vocabularies ##### During class selection @@ -74,9 +74,9 @@ There are two options to add properties: In the _add target class_ dialog you may select different kind of properties: -- class properties - properties defined in the domain of the selected class or its super-classes -- default properties - typical well-known properties like `rdfs:label` or `rdfs:comment` -- generic properties - properties defined with no explicit domain (or in domain of `owl:Thing`) +- class properties - properties defined in the domain of the selected class or its super-classes +- default properties - typical well-known properties like `rdfs:label` or `rdfs:comment` +- generic properties - properties defined with no explicit domain (or in domain of `owl:Thing`) The property preview helps to confirm your choice. @@ -86,8 +86,8 @@ The property preview helps to confirm your choice. The _add property from vocabularies_ dialog allows you to search and select a property and to configure it in the desired way: -- redefine the role of a property, to use a DatatypeProperty in the role of an ObjectProperty, or vice versa -- define the _direction_ an ObjectProperty should be used in +- redefine the role of a property, to use a DatatypeProperty in the role of an ObjectProperty, or vice versa +- define the _direction_ an ObjectProperty should be used in #### Create direct mappings diff --git a/docs/build/reference/aggregator/average.md b/docs/build/reference/aggregator/average.md index 3ca9efb0..21ab406f 100644 --- a/docs/build/reference/aggregator/average.md +++ b/docs/build/reference/aggregator/average.md @@ -2,9 +2,11 @@ title: "Average" description: "Computes the weighted average." icon: octicons/cross-reference-24 -tags: +tags: --- + # Average + @@ -45,4 +47,4 @@ Computes the weighted average. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/aggregator/firstNonEmpty.md b/docs/build/reference/aggregator/firstNonEmpty.md index 3b6165fc..e0fa87d9 100644 --- a/docs/build/reference/aggregator/firstNonEmpty.md +++ b/docs/build/reference/aggregator/firstNonEmpty.md @@ -2,9 +2,11 @@ title: "First non-empty score" description: "Forwards the first input that provides a non-empty similarity score." icon: octicons/cross-reference-24 -tags: +tags: --- + # First non-empty score + @@ -30,4 +32,4 @@ Forwards the first input that provides a non-empty similarity score. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/aggregator/geometricMean.md b/docs/build/reference/aggregator/geometricMean.md index 11bbed8a..b330bca7 100644 --- a/docs/build/reference/aggregator/geometricMean.md +++ b/docs/build/reference/aggregator/geometricMean.md @@ -2,9 +2,11 @@ title: "Geometric mean" description: "Compute the (weighted) geometric mean." icon: octicons/cross-reference-24 -tags: +tags: --- + # Geometric mean + @@ -70,4 +72,4 @@ Compute the (weighted) geometric mean. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/aggregator/handleMissingValues.md b/docs/build/reference/aggregator/handleMissingValues.md index 28ecf268..f465e951 100644 --- a/docs/build/reference/aggregator/handleMissingValues.md +++ b/docs/build/reference/aggregator/handleMissingValues.md @@ -2,9 +2,11 @@ title: "Handle missing values" description: "Generates a default similarity score, if no similarity score is provided (e.g., due to missing values). Using this operator can have a performance impact, since it lowers the efficiency of the underlying computation." icon: octicons/cross-reference-24 -tags: +tags: --- + # Handle missing values + @@ -40,9 +42,9 @@ Generates a default similarity score, if no similarity score is provided (e.g., The default value to be generated, if no similarity score is provided. Must be a value between -1 (inclusive) and 1 (inclusive). '1' represents boolean true and '-1' represents boolean false. -- ID: `defaultValue` -- Datatype: `double` -- Default Value: `-1.0` +* ID: `defaultValue` +* Datatype: `double` +* Default Value: `-1.0` @@ -50,4 +52,4 @@ The default value to be generated, if no similarity score is provided. Must be a ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/aggregator/index.md b/docs/build/reference/aggregator/index.md index f3a30984..8b5789c3 100644 --- a/docs/build/reference/aggregator/index.md +++ b/docs/build/reference/aggregator/index.md @@ -5,7 +5,9 @@ tags: - Build - Reference --- + # Aggregators + This kind of task aggregates multiple similarity scores. diff --git a/docs/build/reference/aggregator/max.md b/docs/build/reference/aggregator/max.md index 9d3a55ef..4c85e998 100644 --- a/docs/build/reference/aggregator/max.md +++ b/docs/build/reference/aggregator/max.md @@ -2,9 +2,11 @@ title: "Or" description: "At least one input score must be within the threshold. Selects the maximum score." icon: octicons/cross-reference-24 -tags: +tags: --- + # Or + @@ -52,4 +54,4 @@ At least one input score must be within the threshold. Selects the maximum score ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/aggregator/min.md b/docs/build/reference/aggregator/min.md index e323f785..0414b254 100644 --- a/docs/build/reference/aggregator/min.md +++ b/docs/build/reference/aggregator/min.md @@ -2,9 +2,11 @@ title: "And" description: "All input scores must be within the threshold. Selects the minimum score." icon: octicons/cross-reference-24 -tags: +tags: --- + # And + @@ -52,4 +54,4 @@ All input scores must be within the threshold. Selects the minimum score. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/aggregator/negate.md b/docs/build/reference/aggregator/negate.md index f1c4974e..297cdc2d 100644 --- a/docs/build/reference/aggregator/negate.md +++ b/docs/build/reference/aggregator/negate.md @@ -2,9 +2,11 @@ title: "Negate" description: "Negates the result of the input comparison. A single input is expected. Using this operator can have a performance impact, since it lowers the efficiency of the underlying computation." icon: octicons/cross-reference-24 -tags: +tags: --- + # Negate + @@ -18,4 +20,4 @@ Negates the result of the input comparison. A single input is expected. Using th ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/aggregator/quadraticMean.md b/docs/build/reference/aggregator/quadraticMean.md index 84e54453..ed5666cd 100644 --- a/docs/build/reference/aggregator/quadraticMean.md +++ b/docs/build/reference/aggregator/quadraticMean.md @@ -2,9 +2,11 @@ title: "Euclidian distance" description: "Calculates the Euclidian distance." icon: octicons/cross-reference-24 -tags: +tags: --- + # Euclidian distance + @@ -78,4 +80,4 @@ Calculates the Euclidian distance. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/aggregator/scale.md b/docs/build/reference/aggregator/scale.md index 6dbc6efb..f8f07718 100644 --- a/docs/build/reference/aggregator/scale.md +++ b/docs/build/reference/aggregator/scale.md @@ -2,9 +2,11 @@ title: "Scale" description: "Scales a similarity score by a factor." icon: octicons/cross-reference-24 -tags: +tags: --- + # Scale + @@ -48,9 +50,9 @@ Scales a similarity score by a factor. All input similarity values are multiplied with this factor. -- ID: `factor` -- Datatype: `double` -- Default Value: `1.0` +* ID: `factor` +* Datatype: `double` +* Default Value: `1.0` @@ -58,4 +60,4 @@ All input similarity values are multiplied with this factor. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/.pages b/docs/build/reference/customtask/.pages index cd2d435a..88ea3f57 100644 --- a/docs/build/reference/customtask/.pages +++ b/docs/build/reference/customtask/.pages @@ -64,6 +64,7 @@ nav: - "Store Vector Embeddings": cmem_plugin_pgvector-Store.md - "Unpivot": Unpivot.md - "Update Graph Insights Snapshots": cmem_plugin_graph_insights-Update.md + - "Update SemSpect": cmem_plugin_semspect-task-Update.md - "Upload File to Knowledge Graph": eccencaDataPlatformGraphStoreFileUploadOperator.md - "Upload files to Nextcloud": cmem_plugin_nextcloud-Upload.md - "Upload local files": cmem_plugin_project_resources-UploadLocalFiles.md diff --git a/docs/build/reference/customtask/CancelWorkflow.md b/docs/build/reference/customtask/CancelWorkflow.md index 79bd3ab6..20a1f5fa 100644 --- a/docs/build/reference/customtask/CancelWorkflow.md +++ b/docs/build/reference/customtask/CancelWorkflow.md @@ -2,10 +2,12 @@ title: "Cancel Workflow" description: "Cancels a workflow if a specified condition is fulfilled. A typical use case for this operator is to cancel the workflow execution if the input data is empty." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Cancel Workflow + @@ -59,4 +61,4 @@ If true, the workflow execution will fail if the condition is met. If false, the ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/ConcatenateToFile.md b/docs/build/reference/customtask/ConcatenateToFile.md index e2ea70db..b0720a3d 100644 --- a/docs/build/reference/customtask/ConcatenateToFile.md +++ b/docs/build/reference/customtask/ConcatenateToFile.md @@ -2,10 +2,12 @@ title: "Concatenate to file" description: "Concatenates values into a file." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Concatenate to file + @@ -88,4 +90,3 @@ File extension of the output file. - Default Value: `.tmp` - diff --git a/docs/build/reference/customtask/CustomSQLExecution.md b/docs/build/reference/customtask/CustomSQLExecution.md index 4ba021ce..966fbae3 100644 --- a/docs/build/reference/customtask/CustomSQLExecution.md +++ b/docs/build/reference/customtask/CustomSQLExecution.md @@ -2,10 +2,12 @@ title: "Spark SQL query" description: "Executes a custom SQL query on the first input Spark dataframe and returns the result as its output." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Spark SQL query + @@ -29,4 +31,4 @@ SQL command. The name of the table in the statement must be 'dataset', regardles ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/DistinctBy.md b/docs/build/reference/customtask/DistinctBy.md index 09f27d60..1f7eac2c 100644 --- a/docs/build/reference/customtask/DistinctBy.md +++ b/docs/build/reference/customtask/DistinctBy.md @@ -2,10 +2,12 @@ title: "Distinct by" description: "Removes duplicated entities based on a user-defined path. Note that this operator does not retain the order of the entities. Since this operator accepts a flexible input schema, it can only be connected to operators that provide a non-flexible output schema. A typical way to achieve this is to place a transform operator before it, which produces a fixed output schema." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Distinct by + @@ -42,4 +44,4 @@ Strategy to resolve duplicates. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/JsonParserOperator.md b/docs/build/reference/customtask/JsonParserOperator.md index 2c13f3fe..77d5f0e9 100644 --- a/docs/build/reference/customtask/JsonParserOperator.md +++ b/docs/build/reference/customtask/JsonParserOperator.md @@ -2,10 +2,12 @@ title: "Parse JSON" description: "Parses an incoming entity as a JSON dataset. Typically, it is used before a transformation task. Takes exactly one input of which only the first entity is processed." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Parse JSON + @@ -59,4 +61,4 @@ Navigate into arrays automatically. If set to false, the `#array` path operator ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/Merge.md b/docs/build/reference/customtask/Merge.md index a2088c74..84c90a67 100644 --- a/docs/build/reference/customtask/Merge.md +++ b/docs/build/reference/customtask/Merge.md @@ -2,10 +2,12 @@ title: "Join tables" description: "Joins a set of inputs into a single table. Expects a list of entity tables and links. All entity tables are joined into the first entity table using the provided links." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Join tables + @@ -19,4 +21,4 @@ Joins a set of inputs into a single table. Expects a list of entity tables and l ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/MultiTableMerge.md b/docs/build/reference/customtask/MultiTableMerge.md index ec668adf..584c0c30 100644 --- a/docs/build/reference/customtask/MultiTableMerge.md +++ b/docs/build/reference/customtask/MultiTableMerge.md @@ -2,10 +2,12 @@ title: "Merge tables" description: "Stores sets of instance and mapping inputs as relational tables with the mapping as an n:m relation. Expects a list of entity tables and links. All entity tables have a relation to the first entity table using the provided links." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Merge tables + @@ -59,4 +61,4 @@ Name of the tables joined to the pivot. Comma separated list. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/Pivot.md b/docs/build/reference/customtask/Pivot.md index 368cde2e..1203e204 100644 --- a/docs/build/reference/customtask/Pivot.md +++ b/docs/build/reference/customtask/Pivot.md @@ -2,10 +2,12 @@ title: "Pivot" description: "The pivot operator takes data in separate rows, aggregates it and converts it into columns." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Pivot + @@ -497,4 +499,4 @@ Prefix to prepend to all generated pivot columns. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/Scheduler.md b/docs/build/reference/customtask/Scheduler.md index 8f99857a..9cdf94ed 100644 --- a/docs/build/reference/customtask/Scheduler.md +++ b/docs/build/reference/customtask/Scheduler.md @@ -2,10 +2,12 @@ title: "Scheduler" description: "Executes a workflow at specified intervals." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Scheduler + @@ -98,4 +100,4 @@ If set to true, this will stop the scheduler, so the failed task is not schedule ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/SearchAddresses.md b/docs/build/reference/customtask/SearchAddresses.md index 509b8ad6..46ef1937 100644 --- a/docs/build/reference/customtask/SearchAddresses.md +++ b/docs/build/reference/customtask/SearchAddresses.md @@ -2,10 +2,12 @@ title: "Search addresses" description: "Looks up locations from textual descriptions using the configured geocoding API. Outputs results as RDF." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Search addresses + @@ -21,18 +23,18 @@ The default configuration is as follows: # url = "https://nominatim.eccenca.com/search" url = "https://photon.komoot.de/api" # url = https://api-adresse.data.gouv.fr/search - + # Additional URL parameters to be attached to all HTTP search requests. Example: '&countrycodes=de&addressdetails=1'. # Will be attached in addition to the parameters set on each search operator directly. searchParameters = "" - + # The minimum pause time between subsequent queries pauseTime = 1s - + # Number of coordinates to be cached in-memory cacheSize = 10 } - + In general, all services adhering to the [Nominatim search API](https://nominatim.org/release-docs/develop/api/Search/) should be usable. Please note that when using public services, the pause time should be set to avoid overloading. @@ -90,4 +92,3 @@ Additional URL parameters to be attached to each HTTP search request. Example: ' - Default Value: `None` - diff --git a/docs/build/reference/customtask/SendEMail.md b/docs/build/reference/customtask/SendEMail.md index d4c5f5fd..916cb821 100644 --- a/docs/build/reference/customtask/SendEMail.md +++ b/docs/build/reference/customtask/SendEMail.md @@ -2,10 +2,12 @@ title: "Send email" description: "Sends an email using an SMTP server." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Send email + @@ -233,4 +235,3 @@ The number of retries per email when send errors are encountered. - Default Value: `2` - diff --git a/docs/build/reference/customtask/SparkFunction.md b/docs/build/reference/customtask/SparkFunction.md index 74ab1ce8..c70847f3 100644 --- a/docs/build/reference/customtask/SparkFunction.md +++ b/docs/build/reference/customtask/SparkFunction.md @@ -2,10 +2,12 @@ title: "Execute Spark function" description: "Applies a specified Scala function to a specified field." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Execute Spark function + @@ -53,4 +55,4 @@ Alias. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/Template.md b/docs/build/reference/customtask/Template.md index b2710aad..ccb1038c 100644 --- a/docs/build/reference/customtask/Template.md +++ b/docs/build/reference/customtask/Template.md @@ -2,10 +2,12 @@ title: "Evaluate template" description: "Evaluates a template on a sequence of entities. Can be used after a transformation or directly after datasets that output a single table, such as CSV or Excel." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Evaluate template + @@ -26,7 +28,7 @@ For each input entity, a output entity is generated that provides a single outpu If 'full evaluation' is enabled, the entire input set will be evaluated at once. -The entities variable will contain all input entities and can be iterated over: +The entities variable will contain all input entities and can be iterated over: {% for entity in entities %} {{entity.property}} @@ -104,4 +106,4 @@ If true, the input attributes will be forwarded to the output. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/Unpivot.md b/docs/build/reference/customtask/Unpivot.md index 55054648..b2d06628 100644 --- a/docs/build/reference/customtask/Unpivot.md +++ b/docs/build/reference/customtask/Unpivot.md @@ -2,10 +2,12 @@ title: "Unpivot" description: "Given a list of table columns, transforms those columns into attribute-value pairs." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Unpivot + @@ -532,4 +534,4 @@ Comma separated list of pivot column names. This property will override all infe ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/XmlParserOperator.md b/docs/build/reference/customtask/XmlParserOperator.md index 408cd753..77422a5c 100644 --- a/docs/build/reference/customtask/XmlParserOperator.md +++ b/docs/build/reference/customtask/XmlParserOperator.md @@ -2,10 +2,12 @@ title: "Parse XML" description: "Takes exactly one input and reads either the defined inputPath or the first value of the first entity as XML document. Then executes the given output entity schema similar to the XML dataset to construct the result entities." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Parse XML + @@ -49,4 +51,4 @@ A URI pattern that is relative to the base URI of the input entity, e.g., /{ID}, ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/addProjectFiles.md b/docs/build/reference/customtask/addProjectFiles.md index b58d93b1..a8fa37fe 100644 --- a/docs/build/reference/customtask/addProjectFiles.md +++ b/docs/build/reference/customtask/addProjectFiles.md @@ -2,10 +2,12 @@ title: "Add project files" description: "Adds file resources to the project that are piped into the input port." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Add project files + @@ -49,4 +51,4 @@ The strategy to use if a file with the same name already exists. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/clearDataset.md b/docs/build/reference/customtask/clearDataset.md index 5292f5e7..04ebd382 100644 --- a/docs/build/reference/customtask/clearDataset.md +++ b/docs/build/reference/customtask/clearDataset.md @@ -2,10 +2,12 @@ title: "Clear dataset" description: "Clears the dataset that is connected to the output of this operator." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Clear dataset + @@ -19,4 +21,4 @@ Clears the dataset that is connected to the output of this operator. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/cmem-plugin-jq-workflow.md b/docs/build/reference/customtask/cmem-plugin-jq-workflow.md index da9ac73d..0281ebf3 100644 --- a/docs/build/reference/customtask/cmem-plugin-jq-workflow.md +++ b/docs/build/reference/customtask/cmem-plugin-jq-workflow.md @@ -2,11 +2,13 @@ title: "jq" description: "Process a JSON document with a jq filter / program." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # jq + !!! note inline end "Python Plugin" @@ -22,7 +24,7 @@ tags: In order to test jq expressions, you can use [play.jqlang.org](https://play.jqlang.org/). -## Basic concepts: +## Basic concepts - Filters separated by a comma will produce multiple independent outputs: `,` - Will ignores error if the type is unexpected: `?` @@ -101,4 +103,3 @@ The jq program to apply to the input JSON string. - Default Value: `None` - diff --git a/docs/build/reference/customtask/cmem_plugin_auth-workflow-auth-OAuth2.md b/docs/build/reference/customtask/cmem_plugin_auth-workflow-auth-OAuth2.md index d77518de..c00fd650 100644 --- a/docs/build/reference/customtask/cmem_plugin_auth-workflow-auth-OAuth2.md +++ b/docs/build/reference/customtask/cmem_plugin_auth-workflow-auth-OAuth2.md @@ -2,11 +2,13 @@ title: "OAuth2 Authentication" description: "Provide an OAuth2 access token for other tasks (via config port)." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # OAuth2 Authentication + !!! note inline end "Python Plugin" @@ -92,4 +94,4 @@ The user account password. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/cmem_plugin_graph_insights-Update.md b/docs/build/reference/customtask/cmem_plugin_graph_insights-Update.md index c2fd760e..770643a1 100644 --- a/docs/build/reference/customtask/cmem_plugin_graph_insights-Update.md +++ b/docs/build/reference/customtask/cmem_plugin_graph_insights-Update.md @@ -2,11 +2,13 @@ title: "Update Graph Insights Snapshots" description: "Update one or more snapshots, optionally selected by affected graph." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Update Graph Insights Snapshots + !!! note inline end "Python Plugin" @@ -35,7 +37,7 @@ This workflow task updates [Graph Insights](https://go.eccenca.com/feature/explo - Graph Insights must be active in your system - User must have permissions to access Graph Insights - The plugin will skip execution with a warning if these conditions are not met - + ## Parameter @@ -62,4 +64,3 @@ Timeout in seconds for Graph Insights API. - Default Value: `100` - diff --git a/docs/build/reference/customtask/cmem_plugin_graphql-workflow-graphql-GraphQLPlugin.md b/docs/build/reference/customtask/cmem_plugin_graphql-workflow-graphql-GraphQLPlugin.md index fe32dbd5..af928251 100644 --- a/docs/build/reference/customtask/cmem_plugin_graphql-workflow-graphql-GraphQLPlugin.md +++ b/docs/build/reference/customtask/cmem_plugin_graphql-workflow-graphql-GraphQLPlugin.md @@ -2,11 +2,13 @@ title: "GraphQL query" description: "Executes a custom GraphQL query to a GraphQL endpoint and saves result to a JSON dataset." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # GraphQL query + !!! note inline end "Python Plugin" @@ -20,7 +22,7 @@ This workflow task performs GraphQL operations by sending in the GraphQL query using, Jinja queries and Jinja variables, which can be obtained from entities. The result of the query is saved as a JSON document in a pre-created JSON dataset. - + ## Parameter @@ -77,4 +79,3 @@ Access token that connects to a GraphQL endpoint to authorize and secure user ac - Default Value: `None` - diff --git a/docs/build/reference/customtask/cmem_plugin_irdi-workflow-irdi_plugin-IrdiPlugin.md b/docs/build/reference/customtask/cmem_plugin_irdi-workflow-irdi_plugin-IrdiPlugin.md index ce18d15a..b4de4a28 100644 --- a/docs/build/reference/customtask/cmem_plugin_irdi-workflow-irdi_plugin-IrdiPlugin.md +++ b/docs/build/reference/customtask/cmem_plugin_irdi-workflow-irdi_plugin-IrdiPlugin.md @@ -2,11 +2,13 @@ title: "Generate base36 IRDIs" description: "Create unique ECLASS IRDIs." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Generate base36 IRDIs + !!! note inline end "Python Plugin" @@ -22,10 +24,10 @@ IRDIs are unique for each combination of (non-advanced) parameters. If no input path is configured, values are read from the URIs of the input (Transformation Input). - All fields of the IRDI are configurable, except `Item Code`, which is created by the plugin. - - Created IRDIs are unique per configuration. + - Created IRDIs are unique per configuration. - Specify a graph that stores the state of Item Codes. - Input and output paths are configurable. - - if no input path is configured, values are read from the URIs of the input + - if no input path is configured, values are read from the URIs of the input (transformation input). @@ -134,4 +136,3 @@ Path from which input values are taken. If empty, values are read from the URIs - Default Value: `None` - diff --git a/docs/build/reference/customtask/cmem_plugin_jira-JqlQuery.md b/docs/build/reference/customtask/cmem_plugin_jira-JqlQuery.md index 89403bf3..55f2439a 100644 --- a/docs/build/reference/customtask/cmem_plugin_jira-JqlQuery.md +++ b/docs/build/reference/customtask/cmem_plugin_jira-JqlQuery.md @@ -2,11 +2,13 @@ title: "JQL query" description: "Search and retrieve JIRA issues." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # JQL query + !!! note inline end "Python Plugin" @@ -32,7 +34,7 @@ for your Atlassian account, to access the API of your atlassian.net hosted Jira ### Jira Server -Base URL of the jira service, e.g. 'https://jira.example.org' +Base URL of the jira service, e.g. '' - ID: `base_url` - Datatype: `string` @@ -113,4 +115,3 @@ Number of items to return per request. - Default Value: `100` - diff --git a/docs/build/reference/customtask/cmem_plugin_kafka-ReceiveMessages.md b/docs/build/reference/customtask/cmem_plugin_kafka-ReceiveMessages.md index 5a8f783a..4bcb53e7 100644 --- a/docs/build/reference/customtask/cmem_plugin_kafka-ReceiveMessages.md +++ b/docs/build/reference/customtask/cmem_plugin_kafka-ReceiveMessages.md @@ -2,11 +2,13 @@ title: "Kafka Consumer (Receive Messages)" description: "Reads messages from a Kafka topic and saves it to a messages dataset (Consumer)." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Kafka Consumer (Receive Messages) + !!! note inline end "Python Plugin" @@ -248,4 +250,3 @@ Setting this to true will disable committing messages after retrival. This means - Default Value: `false` - diff --git a/docs/build/reference/customtask/cmem_plugin_kafka-SendMessages.md b/docs/build/reference/customtask/cmem_plugin_kafka-SendMessages.md index a4e6d197..256d6bb3 100644 --- a/docs/build/reference/customtask/cmem_plugin_kafka-SendMessages.md +++ b/docs/build/reference/customtask/cmem_plugin_kafka-SendMessages.md @@ -2,11 +2,13 @@ title: "Kafka Producer (Send Messages)" description: "Reads a messages dataset and sends records to a Kafka topic (Producer)." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Kafka Producer (Send Messages) + !!! note inline end "Python Plugin" @@ -212,4 +214,3 @@ The compression type for all data generated by the producer. The default is none - Default Value: `none` - diff --git a/docs/build/reference/customtask/cmem_plugin_kubernetes-Execute.md b/docs/build/reference/customtask/cmem_plugin_kubernetes-Execute.md index de8a8b33..28af3caa 100644 --- a/docs/build/reference/customtask/cmem_plugin_kubernetes-Execute.md +++ b/docs/build/reference/customtask/cmem_plugin_kubernetes-Execute.md @@ -2,11 +2,13 @@ title: "Execute a command in a kubernetes pod" description: "Connect to a cluster, execute a command and gather the output." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Execute a command in a kubernetes pod + !!! note inline end "Python Plugin" @@ -21,9 +23,10 @@ This plugin enables execution of commands inside Kubernetes pods and captures th ## Features - Supports multiple connection types: - - **In-cluster**: Uses the service account kubernetes gives to pods + - **In-cluster**: Uses the service account kubernetes gives to pods (for plugins running inside k8s) - - **Explicit config**: Uses a YAML kubeconfig file for external connections + + - **Explicit config**: Uses a YAML kubeconfig file for external connections - Executes shell commands in specified pods within namespaces - Captures both stdout and stderr output - Returns command output as a file entity for further processing @@ -41,7 +44,7 @@ downstream workflow tasks. - Executing maintenance scripts from within or outside the cluster - Gathering system information and logs - Performing health checks and troubleshooting - + ## Parameter @@ -108,4 +111,3 @@ YAML source code of the kube config. - Default Value: `None` - diff --git a/docs/build/reference/customtask/cmem_plugin_llm-CreateEmbeddings.md b/docs/build/reference/customtask/cmem_plugin_llm-CreateEmbeddings.md index 298ae525..a1ecf158 100644 --- a/docs/build/reference/customtask/cmem_plugin_llm-CreateEmbeddings.md +++ b/docs/build/reference/customtask/cmem_plugin_llm-CreateEmbeddings.md @@ -2,11 +2,13 @@ title: "Create Embeddings" description: "Fetch and output LLM created embeddings from input entities." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Create Embeddings + !!! note inline end "Python Plugin" @@ -156,4 +158,3 @@ Changing this value will change the output schema accordingly. Default: _embeddi - Default Value: `_embedding` - diff --git a/docs/build/reference/customtask/cmem_plugin_llm-ExecuteInstructions.md b/docs/build/reference/customtask/cmem_plugin_llm-ExecuteInstructions.md index af6b3893..78ca4f58 100644 --- a/docs/build/reference/customtask/cmem_plugin_llm-ExecuteInstructions.md +++ b/docs/build/reference/customtask/cmem_plugin_llm-ExecuteInstructions.md @@ -2,11 +2,13 @@ title: "Execute Instructions" description: "Send instructions (prompt) to an LLM and process the result." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Execute Instructions + !!! note inline end "Python Plugin" @@ -80,12 +82,14 @@ class StructuredOutput(BaseModel): ## Performance Features Parallel Processing: + - **Concurrent Requests**: Configurable semaphore-controlled API calls - **Batch Processing**: Entities processed in configurable batch sizes - **Rate Limiting**: Optional delays between requests - **Memory Optimization**: Streaming processing with generator patterns Error Handling: + - **Graceful Degradation**: Continue processing on API errors (configurable) - **Detailed Logging**: Comprehensive error reporting and debugging information - **Workflow Integration**: Proper cancellation support and progress reporting @@ -93,11 +97,13 @@ Error Handling: ## API Compatibility Supported Providers: + - **OpenAI**: Direct API access with full feature support - **Azure OpenAI**: Enterprise Azure-hosted services with API versioning - **OpenAI-Compatible**: Anthropic Claude, OpenRouter, local models, and other compatible endpoints Authentication: + - **API Keys**: Secure password-type parameters for API authentication - **Azure Integration**: Supports Azure OpenAI API versioning and endpoint configuration - **Flexible Endpoints**: Custom base URLs for various providers @@ -105,6 +111,7 @@ Authentication: ## Advanced Configuration ### Message Templates + Customize the conversation structure beyond simple prompts: ```json @@ -115,6 +122,7 @@ Customize the conversation structure beyond simple prompts: ``` ### Performance Tuning + - **Temperature Control**: Adjust creativity vs. determinism (0.0-2.0) - **Timeout Management**: Request-level timeout configuration - **Concurrency Limits**: Prevent rate limiting with request throttling @@ -234,6 +242,7 @@ A list of messages comprising the conversation compatible with OpenAI chat compl - ID: `messages_template` - Datatype: `code-json` - Default Value: + ``` json [ { @@ -266,6 +275,7 @@ The Pydantic schema definition with a mandatory class named `StructuredOutput(Ba - ID: `pydantic_schema` - Datatype: `code-python` - Default Value: + ``` python from pydantic import BaseModel @@ -316,5 +326,3 @@ Delay between API requests in seconds to respect rate limits. - Datatype: `double` - Default Value: `0.0` - - diff --git a/docs/build/reference/customtask/cmem_plugin_logpoint-search_logs_task-RetrieveLogs.md b/docs/build/reference/customtask/cmem_plugin_logpoint-search_logs_task-RetrieveLogs.md index f0f25c00..73b9ba58 100644 --- a/docs/build/reference/customtask/cmem_plugin_logpoint-search_logs_task-RetrieveLogs.md +++ b/docs/build/reference/customtask/cmem_plugin_logpoint-search_logs_task-RetrieveLogs.md @@ -2,11 +2,13 @@ title: "Search for Logs" description: "Search and retrieve logs from a Logpoint SIEM system with flexible schema output." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Search for Logs + !!! note inline end "Python Plugin" @@ -31,6 +33,7 @@ to search and retrieve log data based on custom queries. ### Authentication Configure the connection to your Logpoint service using: + - **Service URL**: The base URL of your Logpoint instance - **Username**: Service account username with appropriate permissions - **Secret Key**: API secret key for authentication @@ -64,6 +67,7 @@ Configure the connection to your Logpoint service using: - Output paths: (empty for all fields) Common fields include: + - `source_address`: Source IP address - `destination_address`: Destination IP address - `user`: Username associated with the event @@ -73,6 +77,7 @@ Common fields include: ## Actions Use the plugin actions to explore and configure your searches: + - **Preview output paths**: Executes a test query to see available field paths. - **Preview repositories**: List all accessible log repositories in your Logpoint instance. @@ -151,7 +156,7 @@ Comma-separated list of repository names to search. Leave no trailing comma here ### List of output paths -Comma-separated list of field paths to include in output. Example: 'source_address, destination_address, user, log_ts'. Leave no trailing comma here. If specified, creates a fixed output schema with only these fields. Leave empty for automatic schema detection with all available fields. Use 'Preview output paths' action to discover available field names. See Logpoint field naming conventions at: https://docs.logpoint.com/docs/logpoint-taxonomy-guideline/en/latest/Field%20naming%20convention.html +Comma-separated list of field paths to include in output. Example: 'source_address, destination_address, user, log_ts'. Leave no trailing comma here. If specified, creates a fixed output schema with only these fields. Leave empty for automatic schema detection with all available fields. Use 'Preview output paths' action to discover available field names. See Logpoint field naming conventions at: - ID: `paths_list` - Datatype: `string` @@ -163,4 +168,4 @@ Comma-separated list of field paths to include in output. Example: 'source_addre ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/cmem_plugin_loopwf-task-StartWorkflow.md b/docs/build/reference/customtask/cmem_plugin_loopwf-task-StartWorkflow.md index 7dd1bd94..70cf28e7 100644 --- a/docs/build/reference/customtask/cmem_plugin_loopwf-task-StartWorkflow.md +++ b/docs/build/reference/customtask/cmem_plugin_loopwf-task-StartWorkflow.md @@ -2,11 +2,13 @@ title: "Start Workflow per Entity" description: "Loop over the output of a task and start a sub-workflow for each entity." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Start Workflow per Entity + !!! note inline end "Python Plugin" @@ -116,4 +118,3 @@ When working with file entities, setting this to a proper value will send the fi - Default Value: `None` - diff --git a/docs/build/reference/customtask/cmem_plugin_mattermost.md b/docs/build/reference/customtask/cmem_plugin_mattermost.md index 85126497..a342b3a0 100644 --- a/docs/build/reference/customtask/cmem_plugin_mattermost.md +++ b/docs/build/reference/customtask/cmem_plugin_mattermost.md @@ -2,11 +2,13 @@ title: "Send Mattermost messages" description: "Send messages to Mattermost channels and/or users." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Send Mattermost messages + !!! note inline end "Python Plugin" @@ -46,7 +48,7 @@ input paths are recognized: ### URL -The base URL of your Mattermost deployment. Example: https://mattermost.example.org +The base URL of your Mattermost deployment. Example: - ID: `url` - Datatype: `string` @@ -108,4 +110,4 @@ The message size is limited to a configured maximum (e.g. 16383 characters). ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/cmem_plugin_nextcloud-Download.md b/docs/build/reference/customtask/cmem_plugin_nextcloud-Download.md index 649ede01..4bf4288d 100644 --- a/docs/build/reference/customtask/cmem_plugin_nextcloud-Download.md +++ b/docs/build/reference/customtask/cmem_plugin_nextcloud-Download.md @@ -2,11 +2,13 @@ title: "Download Nextcloud files" description: "Download files from a given Nextcloud instance." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Download Nextcloud files + !!! note inline end "Python Plugin" @@ -23,16 +25,18 @@ directory from which files should be downloaded. Additionally, you may define fi include or exclude specific files within the selected directory. The files are not downloaded to the project resources, but are only available within the workflow itself. -#### Nextcloud List Files input +## Nextcloud List Files input + If this workflow has an input, it will take the data that comes in instead of the selected values. This works only with the schema used in the **Nextcloud List Files** Plugin. Make sure you still add the appropriate URL, identification and token. -#### Important: +### Important + To establish a secure connection, you must generate a dedicated [app-specific password and username](https://docs.nextcloud.com/server/latest/user_manual/de/session_management.html) in the Security section of your Nextcloud account settings. Do not use your standard login credentials. - + ## Parameter @@ -96,7 +100,7 @@ A flag indicating weather an empty output will throw an error. -### Exclude files in subfolders from download. +### Exclude files in subfolders from download A flag indicating whether files located in subfolders should be excluded from the download. @@ -110,4 +114,4 @@ A flag indicating whether files located in subfolders should be excluded from th ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/cmem_plugin_nextcloud-List.md b/docs/build/reference/customtask/cmem_plugin_nextcloud-List.md index 36ec8af9..1a1cee78 100644 --- a/docs/build/reference/customtask/cmem_plugin_nextcloud-List.md +++ b/docs/build/reference/customtask/cmem_plugin_nextcloud-List.md @@ -2,11 +2,13 @@ title: "List Nextcloud files" description: "List directories and files from a given Nextcloud folder." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # List Nextcloud files + !!! note inline end "Python Plugin" @@ -23,11 +25,12 @@ Given the URL of the target Nextcloud instance along with your credentials, you directory from which data should be extracted. Additionally, you may define file patterns to include or exclude specific files within the selected directory. -#### Important: +## Important + To establish a secure connection, you must generate a dedicated [app-specific password and username](https://docs.nextcloud.com/server/latest/user_manual/de/session_management.html) in the Security section of your Nextcloud account settings. Do not use your standard login credentials. - + ## Parameter @@ -91,7 +94,7 @@ A flag indicating weather an empty output will throw an error. -### Exclude files in subfolders. +### Exclude files in subfolders A flag indicating whether files located in subfolders should be excluded from the workflow. @@ -105,4 +108,4 @@ A flag indicating whether files located in subfolders should be excluded from th ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/cmem_plugin_nextcloud-Upload.md b/docs/build/reference/customtask/cmem_plugin_nextcloud-Upload.md index e4c1d3b6..db9afa21 100644 --- a/docs/build/reference/customtask/cmem_plugin_nextcloud-Upload.md +++ b/docs/build/reference/customtask/cmem_plugin_nextcloud-Upload.md @@ -2,11 +2,13 @@ title: "Upload files to Nextcloud" description: "Upload files to a given Nextcloud instance." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Upload files to Nextcloud + !!! note inline end "Python Plugin" @@ -21,15 +23,17 @@ This workflow task uploads files to a specified Nextcloud service instance. Given the URL of the target Nextcloud instance along with your credentials, you can specify any directory to which files should be uploaded. -### Input +## Input + If this workflow has an input, it will take the data that comes in instead of the selected source file. -#### Important: +### Important + To establish a secure connection, you must generate a dedicated [app-specific password and username](https://docs.nextcloud.com/server/latest/user_manual/de/session_management.html) in the Security section of your Nextcloud account settings. Do not use your standard login credentials. - + ## Parameter @@ -85,5 +89,3 @@ The file which you would like to upload. If this is set, no connections in the w - Datatype: `string` - Default Value: `None` - - diff --git a/docs/build/reference/customtask/cmem_plugin_office365-Download.md b/docs/build/reference/customtask/cmem_plugin_office365-Download.md index 45893bc8..6dc544e6 100644 --- a/docs/build/reference/customtask/cmem_plugin_office365-Download.md +++ b/docs/build/reference/customtask/cmem_plugin_office365-Download.md @@ -2,11 +2,13 @@ title: "Download Office 365 Files" description: "Download files from Microsoft OneDrive or Sites" icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Download Office 365 Files + !!! note inline end "Python Plugin" @@ -29,12 +31,12 @@ Admin consent is required to activate these permissions. With this setup, anyone with the secret can access all users' OneDrives and all Sharepoint/Team sites. -#### Important +## Important Make sure only trusted admins can create or manage secrets! Whoever holds the secrets has all the access to granted resources so best not to distribute recklessly. - + ## Parameter @@ -140,5 +142,3 @@ Specifies the maximum number of threads used for parallel execution of the workf - Datatype: `Long` - Default Value: `32` - - diff --git a/docs/build/reference/customtask/cmem_plugin_office365-List.md b/docs/build/reference/customtask/cmem_plugin_office365-List.md index 01d2008a..45e6718c 100644 --- a/docs/build/reference/customtask/cmem_plugin_office365-List.md +++ b/docs/build/reference/customtask/cmem_plugin_office365-List.md @@ -2,11 +2,13 @@ title: "List Office 365 Files" description: "List files from OneDrive or Sites" icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # List Office 365 Files + !!! note inline end "Python Plugin" @@ -29,12 +31,12 @@ Admin consent is required to activate these permissions. With this setup, anyone with the secret can access all users' OneDrives and all Sharepoint/Team sites. -#### Important +## Important Make sure only trusted admins can create or manage secrets! Whoever holds the secrets has all the access to granted resources so best not to distribute recklessly. - + ## Parameter @@ -140,5 +142,3 @@ Specifies the maximum number of threads used for parallel execution of the workf - Datatype: `Long` - Default Value: `32` - - diff --git a/docs/build/reference/customtask/cmem_plugin_office365-Upload.md b/docs/build/reference/customtask/cmem_plugin_office365-Upload.md index ed57773e..8af08922 100644 --- a/docs/build/reference/customtask/cmem_plugin_office365-Upload.md +++ b/docs/build/reference/customtask/cmem_plugin_office365-Upload.md @@ -2,11 +2,13 @@ title: "Office 365 Upload Files" description: "Upload files to OneDrive or a site Sharepoint" icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Office 365 Upload Files + !!! note inline end "Python Plugin" @@ -29,12 +31,12 @@ Admin consent is required to activate these permissions. With this setup, anyone with the secret can access all users' OneDrives and all Sharepoint/Team sites. -#### Important +## Important Make sure only trusted admins can create or manage secrets! Whoever holds the secrets has all the access to granted resources so best not to distribute recklessly. - + ## Parameter @@ -120,5 +122,3 @@ Specifies the maximum number of threads used for parallel execution of the workf - Datatype: `Long` - Default Value: `32` - - diff --git a/docs/build/reference/customtask/cmem_plugin_parameters-ParametersPlugin.md b/docs/build/reference/customtask/cmem_plugin_parameters-ParametersPlugin.md index 93bb0778..b7fd9ddc 100644 --- a/docs/build/reference/customtask/cmem_plugin_parameters-ParametersPlugin.md +++ b/docs/build/reference/customtask/cmem_plugin_parameters-ParametersPlugin.md @@ -2,11 +2,13 @@ title: "Set or Overwrite parameters" description: "Connect this task to a config port of another task in order to set or overwrite the parameter values of this task." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Set or Overwrite parameters + !!! note inline end "Python Plugin" @@ -27,7 +29,7 @@ You can also use multiline values with `|` Example parameter configuration: -``` +```text url: http://example.org method: GET query: | @@ -45,11 +47,12 @@ limit: 5 ### Parameter Configuration -Your parameter configuration in YAML Syntax. One 'parameter: value' pair per line. url: http://example.org method: GET query: | SELECT ?s WHERE {{ ?s ?p ?o }} execute_once: True limit: 5 +Your parameter configuration in YAML Syntax. One 'parameter: value' pair per line. url: method: GET query: | SELECT ?s WHERE {{ ?s ?p ?o }} execute_once: True limit: 5 - ID: `parameters` - Datatype: `code-yaml` - Default Value: + ``` yaml url: http://example.org method: GET @@ -69,4 +72,4 @@ limit: 5 ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/cmem_plugin_pdf_extract-pdf_extract-PdfExtract.md b/docs/build/reference/customtask/cmem_plugin_pdf_extract-pdf_extract-PdfExtract.md index eb44cadd..c76c3fca 100644 --- a/docs/build/reference/customtask/cmem_plugin_pdf_extract-pdf_extract-PdfExtract.md +++ b/docs/build/reference/customtask/cmem_plugin_pdf_extract-pdf_extract-PdfExtract.md @@ -2,11 +2,13 @@ title: "Extract from PDF files" description: "Extract text and tables from PDF files" icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Extract from PDF files + !!! note inline end "Python Plugin" @@ -25,7 +27,7 @@ The output is a JSON string on the path `pdf_extract_output`. The format depends ### Output one entity/value per file -``` +```text { "metadata": { "Filename": "sample.pdf", @@ -52,7 +54,7 @@ The output is a JSON string on the path `pdf_extract_output`. The format depends ### Output one entity/value for all files -``` +```text [ { "metadata": {"Filename": "file1.pdf", ...}, @@ -69,7 +71,7 @@ The output is a JSON string on the path `pdf_extract_output`. The format depends ## Input format This task can either work with project files when a regular expression is being used or with -entities coming from another task or dataset. +entities coming from another task or dataset. The input must be file entities following the [FileEntitySchema](https://github.com/eccenca/cmem-plugin-base/blob/main/cmem_plugin_base/dataintegration/typed_entities/file.py). If a regular expression is set, the input ports will close and no connection will be possible. @@ -92,6 +94,7 @@ If set to "Combine", the results of all files will be combined into a single out **Error Handling Mode** Specifies how errors during PDF extraction should be handled. + - *Ignore*: Log errors and continue processing, returning empty or error-marked results. - *Raise on errors*: Raise an error when extraction fails. - *Raise on errors and warnings*: Treat any warning from the underlying PDF extraction module (pdfplumber) when extracting text and tables from pages as an error if empty results are returned. @@ -101,6 +104,7 @@ Specifies how errors during PDF extraction should be handled. Method used to detect tables in PDF pages. For further explanation click [here](https://github.com/jsvine/pdfplumber/blob/stable/README.md#extracting-tables). Available strategies include: + - *lines*: Uses detected lines in the PDF layout to find table boundaries. - *text*: Relies on text alignment and spacing. - *lattice*: Best for machine-generated perfect grids. @@ -113,9 +117,10 @@ Defines a custom table extraction strategy using YAML syntax. Only used if "cust **Text extraction strategy** -Method used to extract text in PDF pages. For further explanation click [here](https://github.com/jsvine/pdfplumber/blob/stable/README.md#extracting-text). +Method used to extract text in PDF pages. For further explanation click [here](https://github.com/jsvine/pdfplumber/blob/stable/README.md#extracting-text). Available strategies include: + - *default*: Balanced for most digital PDFs. - *raw*: Extract the PDFs with no merging of text fragments. - *scanned*: Best for scanned PDFs as it merges text more agressively. @@ -207,6 +212,7 @@ Custom table extraction strategy in YAML format. - ID: `custom_table_strategy` - Datatype: `multiline string` - Default Value: + ``` text # edge_min_length: 3 # explicit_horizontal_lines: [] @@ -243,6 +249,7 @@ Custom text extraction strategy in YAML format. - ID: `custom_text_strategy` - Datatype: `multiline string` - Default Value: + ``` text # extra_attrs: [] # horizontal_ltr: true @@ -267,5 +274,3 @@ The maximum number of processes to use for processing multiple files concurrentl - Datatype: `Long` - Default Value: `9` - - diff --git a/docs/build/reference/customtask/cmem_plugin_pgvector-Search.md b/docs/build/reference/customtask/cmem_plugin_pgvector-Search.md index 2091e5f7..80f98b6f 100644 --- a/docs/build/reference/customtask/cmem_plugin_pgvector-Search.md +++ b/docs/build/reference/customtask/cmem_plugin_pgvector-Search.md @@ -2,11 +2,13 @@ title: "Search Vector Embeddings" description: "Search for top-k metadata stored in Postgres Vector Store (PGVector)." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Search Vector Embeddings + !!! note inline end "Python Plugin" @@ -148,4 +150,3 @@ The distance strategy to use. (default: COSINE) - Default Value: `COSINE` - diff --git a/docs/build/reference/customtask/cmem_plugin_pgvector-Store.md b/docs/build/reference/customtask/cmem_plugin_pgvector-Store.md index db87b4a0..566ea808 100644 --- a/docs/build/reference/customtask/cmem_plugin_pgvector-Store.md +++ b/docs/build/reference/customtask/cmem_plugin_pgvector-Store.md @@ -2,11 +2,13 @@ title: "Store Vector Embeddings" description: "Store embeddings into Postgres Vector Store (PGVector)." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Store Vector Embeddings + !!! note inline end "Python Plugin" @@ -139,4 +141,3 @@ The number of entries to be processed in batch. - Default Value: `100` - diff --git a/docs/build/reference/customtask/cmem_plugin_project_resources-List.md b/docs/build/reference/customtask/cmem_plugin_project_resources-List.md index eb727dd7..d2046b69 100644 --- a/docs/build/reference/customtask/cmem_plugin_project_resources-List.md +++ b/docs/build/reference/customtask/cmem_plugin_project_resources-List.md @@ -2,11 +2,13 @@ title: "List project files" description: "List file resources from the project." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # List project files + !!! note inline end "Python Plugin" @@ -32,7 +34,7 @@ The regular expression has to match the `fullPath` of the file and is case sensi Given this list of example files of a project: -``` +```text dataset.csv my-dataset.xml json/example.json @@ -70,4 +72,4 @@ The regex for filtering the file names. The regex needs to match the full path ( ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/cmem_plugin_project_resources-UploadLocalFiles.md b/docs/build/reference/customtask/cmem_plugin_project_resources-UploadLocalFiles.md index da87e74a..ac4510bd 100644 --- a/docs/build/reference/customtask/cmem_plugin_project_resources-UploadLocalFiles.md +++ b/docs/build/reference/customtask/cmem_plugin_project_resources-UploadLocalFiles.md @@ -2,11 +2,13 @@ title: "Upload local files" description: "Replace a file dataset resource with a local file or upload multiple local files to a project." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Upload local files + !!! note inline end "Python Plugin" @@ -60,4 +62,3 @@ Which activity should be done with the selected local files. - Default Value: `SEND_TO_TASK` - diff --git a/docs/build/reference/customtask/cmem_plugin_reason-plugin_reason-ReasonPlugin.md b/docs/build/reference/customtask/cmem_plugin_reason-plugin_reason-ReasonPlugin.md index 4e80aa0f..8680cfde 100644 --- a/docs/build/reference/customtask/cmem_plugin_reason-plugin_reason-ReasonPlugin.md +++ b/docs/build/reference/customtask/cmem_plugin_reason-plugin_reason-ReasonPlugin.md @@ -2,11 +2,13 @@ title: "Reason" description: "Performs OWL reasoning." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Reason + !!! note inline end "Python Plugin" @@ -16,7 +18,7 @@ tags: e.g. with cmemc. A task performing OWL reasoning. With an OWL ontology and a data graph as input the reasoning result is written to a specified graph. - + ## Options ### Data graph IRI @@ -40,7 +42,8 @@ The IRI of the output graph for the reasoning result. ### Reasoner -The following reasoner options are supported: +The following reasoner options are supported: + - [ELK](https://code.google.com/p/elk-reasoner/) (elk) - [Expression Materializing Reasoner](http://static.javadoc.io/org.geneontology/expression-materializing-reasoner/0.1.3/org/geneontology/reasoner/ExpressionMaterializingReasoner.html) (emr) - [HermiT](http://www.hermit-reasoner.com/) (hermit) @@ -53,8 +56,10 @@ The following reasoner options are supported: The plugin provides the following parameters to include inferred axiom generators: #### Class axiom generators + - **Class inclusion (rdfs:subClassOf)** The reasoner will infer assertions about the hierarchy of classes, i.e. + `SubClassOf:` statements. If there are classes `Person`, `Student` and `Professor`, such that `Person DisjointUnionOf: Student, Professor` holds, the reasoner will infer `Student SubClassOf: Person`. @@ -62,6 +67,7 @@ Student, Professor` holds, the reasoner will infer `Student SubClassOf: Person`. - **Class equivalence (owl:equivalentClass)** The reasoner will infer assertions about the equivalence of classes, i.e. + `EquivalentTo:` statements. If there are classes `Person`, `Student` and `Professor`, such that `Person DisjointUnionOf: Student, Professor` holds, the reasoner will infer `Person EquivalentTo: Student and Professor`. @@ -69,13 +75,15 @@ Student, Professor` holds, the reasoner will infer `Person EquivalentTo: Student - **Class disjointness (owl:disjointWith)** The reasoner will infer assertions about the disjointness of classes, i.e. + `DisjointClasses:` statements. If there are classes `Person`, `Student` and `Professor`, such that `Person DisjointUnionOf: Student, Professor` holds, the reasoner will infer `DisjointClasses: Student, Professor`. - + - **Data property equivalence (owl:equivalentProperty)** The reasoner will infer axioms about the equivalence of data properties, + i.e. `EquivalentProperties` statements. If there are data properties `identifier` and `enrollmentNumber`, such that `enrollmentNumber SubPropertyOf: identifier` and `identifier SubPropertyOf: enrollmentNumber` holds, the reasoner @@ -84,6 +92,7 @@ will infer `Student EquivalentProperties: identifier, enrollmentNumber`. - **Data property inclusion (rdfs:subPropertyOf)** The reasoner will infer axioms about the hierarchy of data properties, + i.e. `SubPropertyOf:` statements. If there are data properties `identifier`, `studentIdentifier` and `enrollmentNumber`, such that `studentIdentifier SubPropertyOf: identifier` and `enrollmentNumber SubPropertyOf: @@ -91,8 +100,10 @@ studentIdentifier` holds, the reasoner will infer `enrollmentNumber SubPropertyO #### Individual axiom generators + - **Individual class assertions (rdf:type)** The reasoner will infer assertions about the classes of individuals, i.e. + `Types:` statements. Assume, there are classes `Person`, `Student` and `University` as well as the property `enrolledIn`, such that `Student EquivalentTo: Person and enrolledIn some University` holds. For @@ -102,6 +113,7 @@ LeipzigUniversity`, the reasoner will infer `John Types: Student`. - **Individual property assertions** The reasoner will infer assertions about the properties of individuals, + i.e. `Facts:` statements. Assume, there are properties `enrolledIn` and `offers`, such that `enrolled SubPropertyChain: enrolledIn o inverse (offers)` holds. For the individuals `John`and `LeipzigUniversity` with the @@ -110,8 +122,10 @@ KnowledgeRepresentation`, the reasoner will infer `John Facts: enrolledIn Leipz #### Object property axiom generators + - **Object property equivalence (owl:equivalentProperty)** The reasoner will infer assertions about the equivalence of object + properties, i.e. `EquivalentTo:` statements. If there are object properties `hasAlternativeLecture` and `hasSameTopicAs`, such that `hasAlternativeLecture Characteristics: Symmetric` and `hasSameTopicAs InverseOf: @@ -121,6 +135,7 @@ hasSameTopicAs`. - **Object property inversion (owl:inverseOf)** The reasoner will infer axioms about the inversion about object + properties, i.e. `InverseOf:` statements. If there is a object property `hasAlternativeLecture`, such that `hasAlternativeLecture Characteristics: Symmetric` holds, the reasoner will infer `hasAlternativeLecture InverseOf: @@ -129,6 +144,7 @@ hasAlternativeLecture`. - **Object property inclusion (rdfs:subPropertyOf)** The reasoner will infer axioms about the inclusion of object properties, + i.e. `SubPropertyOf:` statements. If there are object properties `enrolledIn`, `studentOf` and `hasStudent`, such that `enrolledIn SubPropertyOf: studentOf` and `enrolledIn InverseOf: hasStudent` holds, the reasoner will infer @@ -137,6 +153,7 @@ SubPropertyOf: studentOf` and `enrolledIn InverseOf: hasStudent` holds, the reas - **Object property ranges (rdfs:range)** The reasoner will infer axioms about the ranges of object properties, + i.e. `Range:` statements. If there are classes `Student` and `Lecture` as wells as object properties `hasStudent` and `enrolledIn`, such that `hasStudent Range: Student and enrolledIn some Lecture` holds, the @@ -145,6 +162,7 @@ reasoner will infer `hasStudent Range: Student`. - **Object property domains (rdfs:domain)** The reasoner will infer axioms about the domains of object + properties, i.e. `Domain:` statements. If there are classes `Person`, `Student` and `Professor` as wells as the object property `hasRoleIn`, such that `Professor SubClassOf: Person`, `Student SubClassOf: Person` and @@ -153,7 +171,7 @@ If there are classes `Person`, `Student` and `Professor` as wells as the object ### Validate OWL2 profiles -Validate the input ontology against OWL profiles (DL, EL, QL, RL, and Full) and annotate the result graph. +Validate the input ontology against OWL profiles (DL, EL, QL, RL, and Full) and annotate the result graph. ### Process valid OWL profiles from input @@ -398,4 +416,3 @@ Maximum heap size for the reasoning process in the DI container. ⚠️ Setting - Default Value: `20` - diff --git a/docs/build/reference/customtask/cmem_plugin_reason-plugin_validate-ValidatePlugin.md b/docs/build/reference/customtask/cmem_plugin_reason-plugin_validate-ValidatePlugin.md index a1d41998..0953df94 100644 --- a/docs/build/reference/customtask/cmem_plugin_reason-plugin_validate-ValidatePlugin.md +++ b/docs/build/reference/customtask/cmem_plugin_reason-plugin_validate-ValidatePlugin.md @@ -2,11 +2,13 @@ title: "Validate OWL consistency" description: "Validates the consistency of an OWL ontology." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Validate OWL consistency + !!! note inline end "Python Plugin" @@ -15,8 +17,8 @@ tags: In order to use it, you need to install it, e.g. with cmemc. -A task validating the consistency of an OWL ontology and generating an explanation if inconsistencies are found. -The plugin outputs the explanation as text in Markdown format on the path "markdown", the ontology IRI on the path +A task validating the consistency of an OWL ontology and generating an explanation if inconsistencies are found. +The plugin outputs the explanation as text in Markdown format on the path "markdown", the ontology IRI on the path "ontology_graph_iri", and, if enabled, the valid OWL2 profiles on the path "valid_profiles" as a comma-separated string. ## Options @@ -31,7 +33,8 @@ If enabled, missing imports (`owl:imports`) in the input graphs are ignored. ### Reasoner -The following reasoner options are supported: +The following reasoner options are supported: + - [ELK](https://code.google.com/p/elk-reasoner/) (elk) - [Expression Materializing Reasoner](http://static.javadoc.io/org.geneontology/expression-materializing-reasoner/0.1.3/org/geneontology/reasoner/ExpressionMaterializingReasoner.html) (emr) - [HermiT](http://www.hermit-reasoner.com/) (hermit) @@ -60,6 +63,7 @@ The filename of the Markdown file with the explanation of inconsistencies. ⚠️ Existing files will be overwritten. ### Stop at inconsistencies + Raise an error if inconsistencies are found. If enabled, the plugin does not output entities. ### Validate OWL2 profiles @@ -67,6 +71,7 @@ Raise an error if inconsistencies are found. If enabled, the plugin does not out Validate the input ontology against OWL profiles (DL, EL, QL, RL, and Full) and annotate the result graph. ### Mode + Mode _inconsistency_ generates an explanation for an inconsistent ontology. Mode _unsatisfiability_ generates explanations for many unsatisfiable classes at once. @@ -188,4 +193,3 @@ Maximum heap size for the reasoning process in the DI container. ⚠️ Setting - Default Value: `20` - diff --git a/docs/build/reference/customtask/cmem_plugin_salesforce-SoqlQuery.md b/docs/build/reference/customtask/cmem_plugin_salesforce-SoqlQuery.md index f13deab9..d730f104 100644 --- a/docs/build/reference/customtask/cmem_plugin_salesforce-SoqlQuery.md +++ b/docs/build/reference/customtask/cmem_plugin_salesforce-SoqlQuery.md @@ -2,11 +2,13 @@ title: "SOQL query (Salesforce)" description: "Executes a custom Salesforce Object Query (SOQL) to return sets of data your organization's Salesforce account." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # SOQL query (Salesforce) + !!! note inline end "Python Plugin" @@ -35,11 +37,14 @@ SOQL Query. By default, this Parse SOQL option is set `True` (enabled). Examples: Retrieve all standard fields from all Lead resources. (without parser validation) -``` + +```text SELECT FIELDS(STANDARD) FROM Lead ``` + Retrieve first name and last name of all Contact resources. (with parser validation) -``` + +```text SELECT Contact.Firstname, Contact.Lastname FROM Contact ``` @@ -101,5 +106,3 @@ In addition to have direct output of the fetched entities of your SOQL query, yo - Datatype: `string` - Default Value: `None` - - diff --git a/docs/build/reference/customtask/cmem_plugin_salesforce-workflow-operations-SobjectCreate.md b/docs/build/reference/customtask/cmem_plugin_salesforce-workflow-operations-SobjectCreate.md index e499263a..d3747ef9 100644 --- a/docs/build/reference/customtask/cmem_plugin_salesforce-workflow-operations-SobjectCreate.md +++ b/docs/build/reference/customtask/cmem_plugin_salesforce-workflow-operations-SobjectCreate.md @@ -2,11 +2,13 @@ title: "Create/Update Salesforce Objects" description: "Manipulate data in your organization's Salesforce account." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Create/Update Salesforce Objects + !!! note inline end "Python Plugin" @@ -28,8 +30,8 @@ configured object type. data model (refer to the [Salesforce Standard Objects list](https://developer.salesforce.com/docs/atlas.en-us.238.0.object_reference.meta/object_reference/sforce_api_objects_list.htm)). - The special path `id` is used to identify an object in Salesforce and switch between update/creation mode, means: - - If there is NO id path available, a new object is created. - - If there IS an id path available, an update is done if the object exists. + - If there is NO id path available, a new object is created. + - If there IS an id path available, an update is done if the object exists. Example: @@ -94,4 +96,4 @@ Salesforce Object API Name ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/cmem_plugin_semspect-task-Update.md b/docs/build/reference/customtask/cmem_plugin_semspect-task-Update.md new file mode 100644 index 00000000..a201a3c4 --- /dev/null +++ b/docs/build/reference/customtask/cmem_plugin_semspect-task-Update.md @@ -0,0 +1,106 @@ +--- +title: "Update SemSpect" +description: "Tell SemSpect to prepare a Knowledge Graph for visualization." +icon: octicons/cross-reference-24 +tags: + - WorkflowTask + - PythonPlugin +--- + +# Update SemSpect + + + +!!! note inline end "Python Plugin" + + This operator is part of a Python Plugin Package. + In order to use it, you need to install it, + e.g. with cmemc. + +Tell SemSpect to prepare a Knowledge Graph for visualization. + +## Parameter + +### The URL of the SemSpect application.This needs to be accessible from 'within' DataIntegration + + + +- ID: `base_url` +- Datatype: `string` +- Default Value: `http://semspect:8080/semspect/` + + + +### The SemSpect database ID. Not existing databases will be created + + + +- ID: `database_id` +- Datatype: `string` +- Default Value: `cmem` + + + +### Knowledge Graph + + + +- ID: `graph` +- Datatype: `string` +- Default Value: `None` + + + + + +## Advanced Parameter + +### The URL of the DataPlatform application.This needs to be accessible from 'within' SemSpect + + + +- ID: `dataplatform_base` +- Datatype: `string` +- Default Value: `None` + + + +### Timeout (in seconds) for the overall indexing activity + + + +- ID: `timeout` +- Datatype: `Long` +- Default Value: `300` + + + +### Timeout (in seconds) for individual Semspect requests + + + +- ID: `request_timeout` +- Datatype: `Long` +- Default Value: `10` + + + +### ignore_proxy: Ignore system settings for HTTP proxies for the requests to semspect + + + +- ID: `ignore_proxy` +- Datatype: `boolean` +- Default Value: `false` + + + +### verify_ssl: If disabled, the plugin will accept any TLS certificate presented by the server and will ignore hostname mismatches and/or expired certificates, which will make the requests vulnerable to man-in-the-middle (MitM) attacks. (use for testing only) + + + +- ID: `verify_ssl` +- Datatype: `boolean` +- Default Value: `true` + + diff --git a/docs/build/reference/customtask/cmem_plugin_shapes-plugin_shapes-ShapesPlugin.md b/docs/build/reference/customtask/cmem_plugin_shapes-plugin_shapes-ShapesPlugin.md index 10379316..3199e09f 100644 --- a/docs/build/reference/customtask/cmem_plugin_shapes-plugin_shapes-ShapesPlugin.md +++ b/docs/build/reference/customtask/cmem_plugin_shapes-plugin_shapes-ShapesPlugin.md @@ -2,11 +2,13 @@ title: "Generate SHACL shapes from data" description: "Generate SHACL node and property shapes from a data graph" icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Generate SHACL shapes from data + !!! note inline end "Python Plugin" @@ -25,9 +27,9 @@ The plugin analyzes an input data graph and creates: - **Node shapes**: One for each class (`rdf:type`) used in the data graph - **Property shapes**: For all properties associated with each class, including: - - Regular object properties (subject → object relationships) - - Inverse object properties (object ← subject relationships, marked with ← prefix) - - Datatype properties (literal values) + - Regular object properties (subject → object relationships) + - Inverse object properties (object ← subject relationships, marked with ← prefix) + - Datatype properties (literal values) ## Output @@ -133,7 +135,7 @@ Import the SHACL shapes graph in the CMEM shapes catalog by adding an `owl:impor ### Fetch namespace prefixes from prefix.cc -Fetch the list of namespace prefixes from https://prefix.cc instead of using the local prefix database. If unavailable, fall back to the local database. Prefixes defined in the Corporate Memory project override database prefixes. Enabling this option exposes your IP address to prefix.cc but no other data is shared. If unsure, keep this option disabled. See https://prefix.cc/about. +Fetch the list of namespace prefixes from instead of using the local prefix database. If unavailable, fall back to the local database. Prefixes defined in the Corporate Memory project override database prefixes. Enabling this option exposes your IP address to prefix.cc but no other data is shared. If unsure, keep this option disabled. See . - ID: `prefix_cc` - Datatype: `boolean` @@ -170,4 +172,3 @@ Add information about the plugin and plugin settings to the shapes graph. - Default Value: `false` - diff --git a/docs/build/reference/customtask/cmem_plugin_splitfile-plugin_splitfile-SplitFilePlugin.md b/docs/build/reference/customtask/cmem_plugin_splitfile-plugin_splitfile-SplitFilePlugin.md index 90ba8e37..33cdf688 100644 --- a/docs/build/reference/customtask/cmem_plugin_splitfile-plugin_splitfile-SplitFilePlugin.md +++ b/docs/build/reference/customtask/cmem_plugin_splitfile-plugin_splitfile-SplitFilePlugin.md @@ -2,11 +2,13 @@ title: "Split file" description: "Split a file into multiple parts with a specified size." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Split file + !!! note inline end "Python Plugin" @@ -128,4 +130,3 @@ The path to the internal projects directory. If "Use internal projects directory - Default Value: `/data/datalake` - diff --git a/docs/build/reference/customtask/cmem_plugin_ssh-Download.md b/docs/build/reference/customtask/cmem_plugin_ssh-Download.md index 90be34b6..c85ff195 100644 --- a/docs/build/reference/customtask/cmem_plugin_ssh-Download.md +++ b/docs/build/reference/customtask/cmem_plugin_ssh-Download.md @@ -2,11 +2,13 @@ title: "Download SSH files" description: "Download files from a given SSH instance" icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Download SSH files + !!! note inline end "Python Plugin" @@ -25,25 +27,28 @@ You can also define a regular expression to include or exclude specific files. There is also an option to prevent files in subfolders from being included. -#### Authentication Methods: -* **Password:** Only the password will be used for authentication. The private key field is +## Authentication Methods + +- **Password:** Only the password will be used for authentication. The private key field is ignored, even if filled. -* **Key:** The private key will be used for authentication. If the key is encrypted, the password +- **Key:** The private key will be used for authentication. If the key is encrypted, the password will be used to decrypt it. -#### Error handling modes: -* **Ignore:** Ignores the permission rights of files and lists downloads all files it has access to. +### Error handling modes + +- **Ignore:** Ignores the permission rights of files and lists downloads all files it has access to. Skips folders and files when there is no correct permission. -* **Warning:** Warns the user about files that the user has no permission rights to. Downloads +- **Warning:** Warns the user about files that the user has no permission rights to. Downloads all other files and skips files folder when there is no correct permission. -* **Error:** Throws an error when there is a single file or folder with incorrect permission rights. +- **Error:** Throws an error when there is a single file or folder with incorrect permission rights. -#### Note: -* If a connection cannot be established within 20 seconds, a timeout occurs. -* Currently supported key types are: RSA, DSS, ECDSA, Ed25519. -* Setting the maximum amount of workers to more than 1 may cause a Channel Exception when +#### Note + +- If a connection cannot be established within 20 seconds, a timeout occurs. +- Currently supported key types are: RSA, DSS, ECDSA, Ed25519. +- Setting the maximum amount of workers to more than 1 may cause a Channel Exception when the amount of files is too large - + ## Parameter @@ -117,7 +122,7 @@ The currently selected path within your SSH instance. Auto-completion starts fro -### Error handling for missing permissions. +### Error handling for missing permissions A choice on how to handle errors concerning the permissions rights.When choosing 'ignore' all files get skipped if the current user has correct permission rights.When choosing 'warning' all files get downloaded however there will be a mention that some of the files are not under the users permissionsif there are any and these get skipped.When choosing 'error' the files will not get downloaded if thereis even a single file the user has no access to. @@ -151,7 +156,7 @@ A regular expression used to define which files will get downloaded. ## Advanced Parameter -### Maximum amount of workers. +### Maximum amount of workers Determines the amount of workers used for concurrent thread execution of the task. Default is 1, maximum is 32. Note that too many workers can cause a ChannelException. @@ -159,5 +164,3 @@ Determines the amount of workers used for concurrent thread execution of the tas - Datatype: `Long` - Default Value: `1` - - diff --git a/docs/build/reference/customtask/cmem_plugin_ssh-Execute.md b/docs/build/reference/customtask/cmem_plugin_ssh-Execute.md index afc2dfa7..f0b306ad 100644 --- a/docs/build/reference/customtask/cmem_plugin_ssh-Execute.md +++ b/docs/build/reference/customtask/cmem_plugin_ssh-Execute.md @@ -2,11 +2,13 @@ title: "Execute commands via SSH" description: "Execute commands on a given SSH instance." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Execute commands via SSH + !!! note inline end "Python Plugin" @@ -21,30 +23,34 @@ This workflow task executes commands on a given SSH instance. By providing the hostname, username, port and authentication method, you can specify the folder in which the command should be executed in. -#### Input Methods: -* **No input:** The command will be executed with no input attached to the plugin. Stdin +## Input Methods + +- **No input:** The command will be executed with no input attached to the plugin. Stdin is non-existent in this case. -* **File input:** The command will be executed with the stdin being represented by the +- **File input:** The command will be executed with the stdin being represented by the files that are connected via the input port of the plugin. This also allows for looping over multiple files executing the same command over them. -#### Output Methods: -* **Structured process output:** The output will produce entities with its own schema including +### Output Methods + +- **Structured process output:** The output will produce entities with its own schema including the stdout and stderr as well as the exit code to confirm the execution of the command. -* **File output:** The stdout will be converted into a file a be provided for further use. -* **No output:** The output port will be closed. +- **File output:** The stdout will be converted into a file a be provided for further use. +- **No output:** The output port will be closed. -#### Authentication Methods: -* **Password:** Only the password will be used for authentication. The private key field is +#### Authentication Methods + +- **Password:** Only the password will be used for authentication. The private key field is ignored, even if filled. -* **Key:** The private key will be used for authentication. If the key is encrypted, the password +- **Key:** The private key will be used for authentication. If the key is encrypted, the password will be used to decrypt it. -#### Note: -* If a connection cannot be established within 20 seconds, a timeout occurs. -* Currently supported key types are: RSA, DSS, ECDSA, Ed25519. - +#### Note + +- If a connection cannot be established within 20 seconds, a timeout occurs. +- Currently supported key types are: RSA, DSS, ECDSA, Ed25519. + ## Parameter @@ -162,4 +168,4 @@ A timeout for the executed command. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/cmem_plugin_ssh-List.md b/docs/build/reference/customtask/cmem_plugin_ssh-List.md index a66642d4..3984c8bc 100644 --- a/docs/build/reference/customtask/cmem_plugin_ssh-List.md +++ b/docs/build/reference/customtask/cmem_plugin_ssh-List.md @@ -2,11 +2,13 @@ title: "List SSH files" description: "List files from a given SSH instance." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # List SSH files + !!! note inline end "Python Plugin" @@ -25,25 +27,28 @@ You can also define a regular expression to include or exclude specific files. There is also an option to prevent files in subfolders from being included. -#### Authentication Methods: -* **Password:** Only the password will be used for authentication. The private key field is +## Authentication Methods + +- **Password:** Only the password will be used for authentication. The private key field is ignored, even if filled. -* **Key:** The private key will be used for authentication. If the key is encrypted, the password +- **Key:** The private key will be used for authentication. If the key is encrypted, the password will be used to decrypt it. -#### Error handling modes: -* **Ignore:** Ignores the permission rights of files and lists them all. Skips folders when there +### Error handling modes + +- **Ignore:** Ignores the permission rights of files and lists them all. Skips folders when there is no correct permission. -* **Warning:** Warns the user about files that the user has no permission rights to. Lists all files +- **Warning:** Warns the user about files that the user has no permission rights to. Lists all files and skips folder when there is no correct permission. -* **Error:** Throws an error when there is a single file or folder with incorrect permission rights. +- **Error:** Throws an error when there is a single file or folder with incorrect permission rights. -#### Note: -* If a connection cannot be established within 20 seconds, a timeout occurs. -* Currently supported key types are: RSA, DSS, ECDSA, Ed25519. -* Setting the maximum amount of workers to more than 1 may cause a Channel Exception when +#### Note + +- If a connection cannot be established within 20 seconds, a timeout occurs. +- Currently supported key types are: RSA, DSS, ECDSA, Ed25519. +- Setting the maximum amount of workers to more than 1 may cause a Channel Exception when the amount of files is too large - + ## Parameter @@ -117,7 +122,7 @@ The currently selected path within your SSH instance. Auto-completion starts fro -### Error handling for missing permissions. +### Error handling for missing permissions A choice on how to handle errors concerning the permissions rights.When choosing 'ignore' all files get listed regardless if the current user has correct permission rightsWhen choosing 'warning' all files get listed however there will be a mention that some of the files are not under the users permissionsif there are anyWhen choosing 'error' the files will not get listed if therethere are files the user has no access to. @@ -151,7 +156,7 @@ A regular expression used to define which files will get listed. ## Advanced Parameter -### Maximum amount of workers. +### Maximum amount of workers Determines the amount of workers used for concurrent thread execution of the task. Default is 1, maximum is 32. Note that too many workers can cause a ChannelException. @@ -159,5 +164,3 @@ Determines the amount of workers used for concurrent thread execution of the tas - Datatype: `Long` - Default Value: `1` - - diff --git a/docs/build/reference/customtask/cmem_plugin_ssh-Upload.md b/docs/build/reference/customtask/cmem_plugin_ssh-Upload.md index 580e16f4..edd1d721 100644 --- a/docs/build/reference/customtask/cmem_plugin_ssh-Upload.md +++ b/docs/build/reference/customtask/cmem_plugin_ssh-Upload.md @@ -2,11 +2,13 @@ title: "Upload SSH files" description: "Upload files to a given SSH instance." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Upload SSH files + !!! note inline end "Python Plugin" @@ -21,16 +23,18 @@ This workflow task uploads files to a given SSH instance. By providing the hostname, username, port and authentication method, you can specify the folder the data should be uploaded to. -#### Authentication Methods: -* **Password:** Only the password will be used for authentication. The private key field is +## Authentication Methods + +- **Password:** Only the password will be used for authentication. The private key field is ignored, even if filled. -* **Key:** The private key will be used for authentication. If the key is encrypted, the password +- **Key:** The private key will be used for authentication. If the key is encrypted, the password will be used to decrypt it. -#### Note: -* If a connection cannot be established within 20 seconds, a timeout occurs. -* Currently supported key types are: RSA, DSS, ECDSA, Ed25519. - +### Note + +- If a connection cannot be established within 20 seconds, a timeout occurs. +- Currently supported key types are: RSA, DSS, ECDSA, Ed25519. + ## Parameter @@ -108,4 +112,4 @@ The currently selected path within your SSH instance. Auto-completion starts fro ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/cmem_plugin_validation-validate-ValidateEntities.md b/docs/build/reference/customtask/cmem_plugin_validation-validate-ValidateEntities.md index 6fcaeecb..0a60e07f 100644 --- a/docs/build/reference/customtask/cmem_plugin_validation-validate-ValidateEntities.md +++ b/docs/build/reference/customtask/cmem_plugin_validation-validate-ValidateEntities.md @@ -2,11 +2,13 @@ title: "Validate Entities" description: "Use a JSON schema to validate entities or a JSON dataset." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Validate Entities + !!! note inline end "Python Plugin" @@ -24,14 +26,14 @@ JSON Schema specification. The used JSON Schema needs to be provided as a JSON Dataset in the project. -### Input Modes +## Input Modes The plugin supports two input modes for validation: 1. **Validate Entities**: Validates entities received from the input port in the workflow. 2. **Validate JSON Dataset**: Validates a JSON dataset stored in the project. - - If the JSON dataset is a JSON array, the schema will validate each object inside the array. - - If the JSON dataset is a JSON object, it will be validated against the schema directly. + - If the JSON dataset is a JSON array, the schema will validate each object inside the array. + - If the JSON dataset is a JSON object, it will be validated against the schema directly. Validated data objects can be sent to an output port for further processing in the workflow or saved in a JSON dataset in the project. @@ -118,5 +120,3 @@ This dataset will be used to store the valid JSON objects after validation. - Datatype: `string` - Default Value: `None` - - diff --git a/docs/build/reference/customtask/cmem_plugin_validation-validate-ValidateGraph.md b/docs/build/reference/customtask/cmem_plugin_validation-validate-ValidateGraph.md index ee12d181..506b4ae4 100644 --- a/docs/build/reference/customtask/cmem_plugin_validation-validate-ValidateGraph.md +++ b/docs/build/reference/customtask/cmem_plugin_validation-validate-ValidateGraph.md @@ -2,11 +2,13 @@ title: "Validate Knowledge Graph" description: "Use SHACL shapes to validate resources in a Knowledge Graph." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Validate Knowledge Graph + !!! note inline end "Python Plugin" @@ -93,6 +95,7 @@ The query to select the resources to validate. Use {{context_graph}} as a placeh - ID: `sparql_query` - Datatype: `code-sparql` - Default Value: + ``` sparql SELECT DISTINCT ?resource FROM <{{context_graph}}> @@ -100,5 +103,3 @@ WHERE { ?resource a ?class . FILTER isIRI(?resource) } ``` - - diff --git a/docs/build/reference/customtask/cmem_plugin_wfreports_get_report.md b/docs/build/reference/customtask/cmem_plugin_wfreports_get_report.md index 234c0359..de3a4703 100644 --- a/docs/build/reference/customtask/cmem_plugin_wfreports_get_report.md +++ b/docs/build/reference/customtask/cmem_plugin_wfreports_get_report.md @@ -2,11 +2,13 @@ title: "Get workflow report" description: "Output a workflow execution report as a JSON file." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Get workflow report + !!! note inline end "Python Plugin" @@ -36,6 +38,7 @@ The plugin outputs a single JSON file entity containing the complete workflow ex ## Usage This operator is useful for: + - Monitoring workflow execution results and failures - Debugging recent errors or warnings - Creating audit trails of workflow runs @@ -79,4 +82,4 @@ The time period in hours of the workflow execution reports that can be listed.Al ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/cmem_plugin_yaml-parse.md b/docs/build/reference/customtask/cmem_plugin_yaml-parse.md index a685dc54..3d63dad1 100644 --- a/docs/build/reference/customtask/cmem_plugin_yaml-parse.md +++ b/docs/build/reference/customtask/cmem_plugin_yaml-parse.md @@ -2,11 +2,13 @@ title: "Parse YAML" description: "Parses files, source code or input values as YAML documents." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Parse YAML + !!! note inline end "Python Plugin" @@ -111,4 +113,3 @@ In case of source mode 'entities', you can specify the requested input path. - Default Value: `text` - diff --git a/docs/build/reference/customtask/combine-csv.md b/docs/build/reference/customtask/combine-csv.md index a94d12b9..e130f1e2 100644 --- a/docs/build/reference/customtask/combine-csv.md +++ b/docs/build/reference/customtask/combine-csv.md @@ -2,11 +2,13 @@ title: "Combine CSV files" description: "Combine CSV files with the same structure to one dataset." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # Combine CSV files + !!! note inline end "Python Plugin" @@ -74,4 +76,4 @@ Stop the workflow if no input files are found or all input files are empty. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/deleteProjectFiles.md b/docs/build/reference/customtask/deleteProjectFiles.md index 4b5c562c..749a805e 100644 --- a/docs/build/reference/customtask/deleteProjectFiles.md +++ b/docs/build/reference/customtask/deleteProjectFiles.md @@ -2,10 +2,12 @@ title: "Delete project files" description: "Removes file resources from the project based on a regular expression." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Delete project files + @@ -16,7 +18,7 @@ The project-relative path of each file of the current project is tested against Given this list of example files of a project: -``` +```text dataset.csv my-dataset.xml json/example.json @@ -60,4 +62,4 @@ If enabled the operator outputs entities, one entity for each deleted file, with ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/downloadFile.md b/docs/build/reference/customtask/downloadFile.md index cda7261a..37ac891a 100644 --- a/docs/build/reference/customtask/downloadFile.md +++ b/docs/build/reference/customtask/downloadFile.md @@ -2,18 +2,22 @@ title: "Download file" description: "Downloads a file from a given URL." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Download file + ## 1. Purpose + The **Download File** operator downloads a single file from a remote URL and exposes it as a file output that can be consumed by downstream operators. Typical use cases: + - Importing external datasets into a workflow. - Downloading configuration files or archives. - Fetching files from internal HTTP endpoints. @@ -21,16 +25,19 @@ Typical use cases: ## 2. Input and output ### Input + - **Inputs:** none The operator does not consume any upstream entities. ### Output + - **Output:** one file - A single file is created from the HTTP response body. - The file is exposed as a file output that downstream operators can read. - The MIME type is taken from the HTTP response `Content-Type` if available. ## 3. Configuration notes + The detailed list of parameters is shown in the UI (auto-generated). This section explains how to think about the most important ones. - **URL** @@ -53,7 +60,9 @@ The detailed list of parameters is shown in the UI (auto-generated). This sectio - The value is stored encrypted in the backend. ## 4. Behaviour + When executed, the operator: + 1. Builds an HTTP request from the configured parameters. 2. Sends an HTTP **GET** request to the configured URL. 3. Streams the response body into a temporary file on disk. @@ -61,6 +70,7 @@ When executed, the operator: 5. Reports the execution via the standard task/execution reporting mechanisms. File handling: + - The file is created with a temporary name. - The file extension is determined as follows: - For ZIP-like content types (e.g. `application/zip`), the extension `.zip` is used. @@ -70,6 +80,7 @@ File handling: Only a single file is produced per execution. If the request fails, no file is emitted. ## 5. Supported URLs and protocols + The operator sends an HTTP request to the configured URL. - **Intended behaviour** @@ -83,7 +94,9 @@ The operator sends an HTTP request to the configured URL. If reliable FTP or other protocols are needed, they should be handled by a dedicated operator or external tooling. ## 6. Error handling and failure modes + Typical failure scenarios: + - **Invalid URL / DNS / connection issues** - The operator fails the execution; no file is produced. - **Non-2xx HTTP status codes (e.g. 404, 500)** @@ -98,21 +111,25 @@ Errors are reported via the standard task and execution reporting mechanisms. ## 7. Examples ### 7.1 Simple HTTP download + - **URL:** `https://example.com/data.csv` - **Parameters:** - Accept header: `text/csv` (optional) - Timeouts: defaults or slightly increased for large files. Result: + - One file containing the downloaded CSV data, which can be passed to downstream file-processing operators. ### 7.2 Authenticated download + - **URL:** `https://internal.example.com/report.json` - **Parameters:** - Authorization header: `Authorization` - Authorization value: `Bearer ` Result: + - One JSON file containing the report, assuming the token is valid and the server returns a successful response. @@ -202,4 +219,4 @@ The authorization header value. Usually this has the form 'type secret', e.g. fo ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/eccencaDataPlatformGraphStoreFileUploadOperator.md b/docs/build/reference/customtask/eccencaDataPlatformGraphStoreFileUploadOperator.md index d75f756a..1bcce04b 100644 --- a/docs/build/reference/customtask/eccencaDataPlatformGraphStoreFileUploadOperator.md +++ b/docs/build/reference/customtask/eccencaDataPlatformGraphStoreFileUploadOperator.md @@ -2,10 +2,12 @@ title: "Upload File to Knowledge Graph" description: "Uploads an N-Triples or Turtle (limited support) file from the file repository to a 'Knowledge Graph' dataset. The output of this operatorcan be the input of datasets that support graph store file upload, e.g. 'Knowledge Graph'. The file will be uploaded to the graph specified in that dataset." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Upload File to Knowledge Graph + @@ -49,4 +51,4 @@ The MIME type of the serialization format of the RDF file. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/eccencaRestOperator.md b/docs/build/reference/customtask/eccencaRestOperator.md index 4a4362d4..a058db2b 100644 --- a/docs/build/reference/customtask/eccencaRestOperator.md +++ b/docs/build/reference/customtask/eccencaRestOperator.md @@ -2,10 +2,12 @@ title: "Execute REST requests" description: "REST operator that fetches and optionally merges data from a REST endpoint. It supports executing multiple requests either via input entities that each overwrite config parameters or via paging. If you only need to download a single file, the 'Download file' operator might be the better option. Most features are currently only supported for JSON REST APIs. From multiple requests the REST operator can produce a merged JSON result, i.e. for JSON it will concatenate all results in a JSON array. Alternatively multiple results can be written directly to file (of a JSON dataset), either as a merged JSON file or one file per request inside a ZIP file. By default the output of this operator is an entity with a single property 'result', which is the (concatenated) JSON string." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Execute REST requests + @@ -13,7 +15,7 @@ tags: ## Core parameter overview - `URL`: The URL the request will be executed against. This value can be overwritten at execution time when the 'Read parameters from input' option - is enabled. This value will also be adapted when a paging approach is configured, see the paging section for more details. + is enabled. This value will also be adapted when a paging approach is configured, see the paging section for more details. - `Method`: One of the following HTTP methods: GET, POST, PUT, PATCH or DELETE. - `Accept`: The ACCEPT header value for content negotiation, e.g. 'application/json'. - `Content type`: The CONTENT-TYPE header value. This is usually used for POST, PUT or PATCH requests when the API endpoint @@ -61,10 +63,11 @@ to fetch all results. This is currently only supported for JSON requests. In both cases the path to the next page value in the response JSON must be defined via the 'Next page JSON path' parameter. In case of the 'Next page identifier' paging method, also the parameter 'Next page ID query parameter' must be set. + - `Next page JSON path`: The property path in the result JSON where the 'next page' URL/value is provided. E.g. for following response structure, the value for this parameter would be `paging/next`: - ``` + ```text { ..., "paging": { @@ -72,6 +75,7 @@ to fetch all results. This is currently only supported for JSON requests. } } ``` + - `Next page ID query parameter`: If the paging method is 'Next page identifier', this defines the query parameter name that should be attached to the original request URL in combination with the 'next page' value of the current response in order to request the next page. @@ -79,11 +83,12 @@ to fetch all results. This is currently only supported for JSON requests. ## Setting HTTP headers - `HTTP headers`: This parameter allows to set HTTP headers of the request being made. Each line of the multi-line value should contain a single header, e.g. - ``` + + ```text Accept-Language: en-US,en;q=0.5 Cache-Control: max-age=0 ``` - + ## Sending a multipart HTTP file request If the content of a POST request should be sent as file content of a multipart HTTP request, instead of the request body, @@ -92,7 +97,7 @@ following parameter must be configured: - `Multi-part file parameter`: If set to a non-empty value then, instead of a normal POST request, a multipart/form-data file upload request will be executed. The value of this parameter is used as the form parameter name. - + ## Output options By default, the response body of a request is output as value of the 'result' property of a single output entity. @@ -104,13 +109,13 @@ a single, merged file (only supported for JSON) or to a ZIP archive, i.e. a file In the latter case an entry per request is added to the ZIP file. Currently, the following datasets support the processing of ZIP files: JSON, XML, CSV and RDF file. -- `Output result as file`: If enabled, instead of outputting a single entity, the result/s will be written directly +- `Output result as file`: If enabled, instead of outputting a single entity, the result/s will be written directly to the file of the file-based dataset that is connected to the output of this operator. If the option 'Read parameters from input' is enabled, it is currently always assumed that multiple requests will be sent. The responses must either be JSON, then the results are merged into a JSON array or the 'Output result as file' option must be enabled in order to write a merged JSON or ZIP file. - + ## Fine-tuning timeouts If requests can take a much longer time than what can usually be expected, it is possible to increase the timeouts to @@ -136,7 +141,7 @@ Following parameters can be tuned in order to decide when an execution should be request configuration as failed. Default: `3` - `Abort when request fails`: When enabled, if a single request configuration eventually fails, i.e. it reaches its max. retry count, the overall execution of the REST operator will fail. -- `Max failed requests`: If set to a value greater 0, the execution will abort if more than the given number of request configurations +- `Max failed requests`: If set to a value greater 0, the execution will abort if more than the given number of request configurations have failed (reached max. retries). This can be used if a number of failed requests can be tolerated. When 'Abort when request fails' is enabled, this option is ignored. @@ -395,4 +400,4 @@ If this is non-empty, a property is created in the root JSON object (if it exist ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/getProjectFiles.md b/docs/build/reference/customtask/getProjectFiles.md index 430afc5f..d86247a3 100644 --- a/docs/build/reference/customtask/getProjectFiles.md +++ b/docs/build/reference/customtask/getProjectFiles.md @@ -2,10 +2,12 @@ title: "Get project files" description: "Get file resources from the project." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Get project files + @@ -49,4 +51,4 @@ Optional MIME type to assign to all retrieved files. If left empty, the MIME typ ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/index.md b/docs/build/reference/customtask/index.md index 0ecaca92..ba2181d6 100644 --- a/docs/build/reference/customtask/index.md +++ b/docs/build/reference/customtask/index.md @@ -5,7 +5,9 @@ tags: - Build - Reference --- + # Custom Workflow Tasks + A custom workflow task is an operator that can be used in a workflow. @@ -78,6 +80,7 @@ A custom workflow task is an operator that can be used in a workflow. | [Store Vector Embeddings](cmem_plugin_pgvector-Store.md) | Store embeddings into Postgres Vector Store (PGVector). | | [Unpivot](Unpivot.md) | Given a list of table columns, transforms those columns into attribute-value pairs. | | [Update Graph Insights Snapshots](cmem_plugin_graph_insights-Update.md) | Update one or more snapshots, optionally selected by affected graph. | + | [Update SemSpect](cmem_plugin_semspect-task-Update.md) | Tell SemSpect to prepare a Knowledge Graph for visualization. | | [Upload File to Knowledge Graph](eccencaDataPlatformGraphStoreFileUploadOperator.md) | Uploads an N-Triples or Turtle (limited support) file from the file repository to a 'Knowledge Graph' dataset. The output of this operatorcan be the input of datasets that support graph store file upload, e.g. 'Knowledge Graph'. The file will be uploaded to the graph specified in that dataset. | | [Upload files to Nextcloud](cmem_plugin_nextcloud-Upload.md) | Upload files to a given Nextcloud instance. | | [Upload local files](cmem_plugin_project_resources-UploadLocalFiles.md) | Replace a file dataset resource with a local file or upload multiple local files to a project. | diff --git a/docs/build/reference/customtask/setParameters.md b/docs/build/reference/customtask/setParameters.md index d6e7a395..e068f627 100644 --- a/docs/build/reference/customtask/setParameters.md +++ b/docs/build/reference/customtask/setParameters.md @@ -2,10 +2,12 @@ title: "Set parameters" description: "Set and overwrite parameters of a task." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Set parameters + @@ -39,4 +41,4 @@ The parameters to set. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/shacl-pyshacl.md b/docs/build/reference/customtask/shacl-pyshacl.md index e2da94a4..4f4812c6 100644 --- a/docs/build/reference/customtask/shacl-pyshacl.md +++ b/docs/build/reference/customtask/shacl-pyshacl.md @@ -2,11 +2,13 @@ title: "SHACL validation with pySHACL" description: "Performs SHACL validation with pySHACL." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask - PythonPlugin --- + # SHACL validation with pySHACL + !!! note inline end "Python Plugin" @@ -133,7 +135,7 @@ If enabled along with the `Add labels` option, `rdfs:label` triples are added fo -### Add shui:conforms flag to focus node resources. +### Add shui:conforms flag to focus node resources If enabled, `shui:conforms false` triples are added to the focus nodes in the validation graph. @@ -183,7 +185,7 @@ Enable SHACL-JS features. -### Remove graph type http://rdfs.org/ns/void#Dataset from data graph +### Remove graph type from data graph Before validating, remove the triple ` a ` from the in-memory data graph. @@ -193,7 +195,7 @@ Before validating, remove the triple ` a from data graph Before validating, remove the triple ` a ` from the in-memory data graph. @@ -203,7 +205,7 @@ Before validating, remove the triple ` a from data graph Before validating, remove the triple ` a ` from the in-memory data graph. @@ -222,4 +224,3 @@ specify a custom max-evaluation-depth. If you find yourself with a legitimate us - Default Value: `15` - diff --git a/docs/build/reference/customtask/sparqlCopyOperator.md b/docs/build/reference/customtask/sparqlCopyOperator.md index 16ad74b1..dadf3ba5 100644 --- a/docs/build/reference/customtask/sparqlCopyOperator.md +++ b/docs/build/reference/customtask/sparqlCopyOperator.md @@ -2,10 +2,12 @@ title: "SPARQL Construct query" description: "A task that executes a SPARQL Construct query on a SPARQL enabled data source and outputs the SPARQL result. If the result should be written to the same RDF store it is read from, the SPARQL Update operator is preferable." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # SPARQL Construct query + @@ -39,4 +41,4 @@ When copying directly to the same SPARQL Endpoint or when copying large amounts ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/sparqlSelectOperator.md b/docs/build/reference/customtask/sparqlSelectOperator.md index f34e8ba4..c541793d 100644 --- a/docs/build/reference/customtask/sparqlSelectOperator.md +++ b/docs/build/reference/customtask/sparqlSelectOperator.md @@ -2,10 +2,12 @@ title: "SPARQL Select query" description: "A task that executes a SPARQL Select query on a SPARQL enabled data source and outputs the SPARQL result. If the SPARQL source is defined on a specific graph, a FROM clause will be added to the query at execution time, except when there already exists a GRAPH or FROM clause in the query. FROM NAMED clauses are not injected." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # SPARQL Select query + @@ -80,4 +82,4 @@ SPARQL query timeout (select/update) in milliseconds. A value of zero means that ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/sparqlUpdateOperator.md b/docs/build/reference/customtask/sparqlUpdateOperator.md index ab70e847..0cfa177b 100644 --- a/docs/build/reference/customtask/sparqlUpdateOperator.md +++ b/docs/build/reference/customtask/sparqlUpdateOperator.md @@ -2,10 +2,12 @@ title: "SPARQL Update query" description: "A task that outputs SPARQL Update queries for every entity from the input based on a SPARQL Update template. The output of this operator should be connected to the SPARQL datasets to which the results should be written." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # SPARQL Update query + @@ -31,7 +33,7 @@ the dollar sign (`$`), i.e. filling in input values via placeholders in the temp ### Example of the `Simple` mode -``` +```text DELETE DATA { ${} rdf:label ${"PROP_FROM_ENTITY_SCHEMA2"} } INSERT DATA { ${} rdf:label ${"PROP_FROM_ENTITY_SCHEMA3"} } ``` @@ -46,7 +48,7 @@ can combine variable substitutions with fixed expressions to construct semi-flex ### Example of the `Velocity Engine` mode -``` +```text DELETE DATA { $row.uri("PROP_FROM_ENTITY_SCHEMA1") rdf:label $row.plainLiteral("PROP_FROM_ENTITY_SCHEMA2") } #if ( $row.exists("PROP_FROM_ENTITY_SCHEMA1") ) INSERT DATA { $row.uri("PROP_FROM_ENTITY_SCHEMA1") rdf:label $row.plainLiteral("PROP_FROM_ENTITY_SCHEMA3") } @@ -82,7 +84,7 @@ Both `inputProperties` and `outputProperties` support the same methods as the `r - `rawUnsafe(inputPath: String)` - `exists(inputPath: String)` -For more information about the Velocity Engine, visit http://velocity.apache.org. +For more information about the Velocity Engine, visit . ### Internal Specifics @@ -125,4 +127,4 @@ The templating mode for the template engine. The possible values are `Simple` an ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/sqlUpdateQueryOperator.md b/docs/build/reference/customtask/sqlUpdateQueryOperator.md index b54d27c9..302173d0 100644 --- a/docs/build/reference/customtask/sqlUpdateQueryOperator.md +++ b/docs/build/reference/customtask/sqlUpdateQueryOperator.md @@ -2,10 +2,12 @@ title: "SQL Update query" description: "A task that outputs SQL queries. The output of this operator should be connected to a remote SQL endpoint on which queries should be executed." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # SQL Update query + @@ -30,4 +32,4 @@ The SQL query to be executed ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/tripleRequestOperator.md b/docs/build/reference/customtask/tripleRequestOperator.md index bc30f9df..301f6ce9 100644 --- a/docs/build/reference/customtask/tripleRequestOperator.md +++ b/docs/build/reference/customtask/tripleRequestOperator.md @@ -2,10 +2,12 @@ title: "Request RDF triples" description: "A task that requests all triples from an RDF dataset." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Request RDF triples + @@ -19,4 +21,4 @@ A task that requests all triples from an RDF dataset. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/ucumNormalizationTask.md b/docs/build/reference/customtask/ucumNormalizationTask.md index 729d7f92..baac8b54 100644 --- a/docs/build/reference/customtask/ucumNormalizationTask.md +++ b/docs/build/reference/customtask/ucumNormalizationTask.md @@ -2,10 +2,12 @@ title: "Normalize units of measurement" description: "Custom task that will substitute numeric values and pertaining unit symbols with a SI-system-unit normalized representation." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Normalize units of measurement + @@ -88,6 +90,7 @@ While all SI units and decimal prefixes are supported by default, custom or obso - ID: `configuration` - Datatype: `multiline string` - Default Value: + ``` text # Example configuration, don't forget to remove the '#' in front of each row. @@ -103,7 +106,7 @@ While all SI units and decimal prefixes are supported by default, custom or obso # Denier , true , den , g/(9.km) # Grain , true , gr , (45.g)/100 # Pound , true , lb , (45359237.kg)/100000000 , # , lbm - + ``` @@ -112,4 +115,4 @@ While all SI units and decimal prefixes are supported by default, custom or obso ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/validateXsdOperator.md b/docs/build/reference/customtask/validateXsdOperator.md index 923bc126..b1192220 100644 --- a/docs/build/reference/customtask/validateXsdOperator.md +++ b/docs/build/reference/customtask/validateXsdOperator.md @@ -2,10 +2,12 @@ title: "Validate XML" description: "Validates an XML dataset against a provided XML schema (XSD) file. Any errors are written to the output. Can be used in conjunction with the `Cancel Workflow` operator in order to stop the workflow if errors have been found." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # Validate XML + @@ -29,4 +31,4 @@ The XSD file to be used for validating the XML. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/customtask/xsltOperator.md b/docs/build/reference/customtask/xsltOperator.md index b1c7477f..05885999 100644 --- a/docs/build/reference/customtask/xsltOperator.md +++ b/docs/build/reference/customtask/xsltOperator.md @@ -2,10 +2,12 @@ title: "XSLT" description: "A task that converts an XML resource via an XSLT script and writes the transformed output into a file resource." icon: octicons/cross-reference-24 -tags: +tags: - WorkflowTask --- + # XSLT + @@ -19,19 +21,20 @@ The **output** of the XML transformation is saved as an output file resource. In In essence and from a technical point of view, the `xsltOperator` is simply a wrapper around the XSLT processor provided by [Saxonica](https://www.saxonica.com/products/products.xml). If you are well-versed in the XSL ecosystem, this is everything you need to know. If not, the remaining of the documentation provides some amount of information and detail on the parts of XSL and XSLT which are relevant for our purposes. + ## Description of XSL and XSLT ### The XSL ecosystem The acronym **XSL** stands for "eXtensible Stylesheet Language". XSL is not a single technology or specification, but a _family of languages_ for processing (transforming) and rendering (presenting) XML documents. It consists of three parts: - 1. XSLT: XSL Transformations - 2. XPath: XML Path Language - 3. XSL-FO: XSL Formatting Objects + 1. XSLT: XSL Transformations + 2. XPath: XML Path Language + 3. XSL-FO: XSL Formatting Objects In a nutshell, this is simply the separation of concerns between "processing" XML and "rendering" the results. -The most relevant of these parts for us, is **XSLT**. XSLT is a language for *transforming* or *processing* XML documents. Originally (around 1999), XSLT was designed for _styling_ XML documents, which is still seen in the nomenclature, e.g. in the term `` or in the acronyms "XSL" and "XSLT" themselves. But other than just _styling XML markup_, XSLT 2.0 (and beyond) is a Turing-complete language, which is used for **transforming XML _data_**. The modern perspective and understanding is therefore not on "XML as a markup language for documents which are presented to a web browser, or converted to a form suitable for printing, such as PDF or PostScript", but, in general terms, on "XML as a means to represent (highly-structured) data, which can be arbitrarily transformed by XSLT". +The most relevant of these parts for us, is **XSLT**. XSLT is a language for _transforming_ or _processing_ XML documents. Originally (around 1999), XSLT was designed for _styling_ XML documents, which is still seen in the nomenclature, e.g. in the term `` or in the acronyms "XSL" and "XSLT" themselves. But other than just _styling XML markup_, XSLT 2.0 (and beyond) is a Turing-complete language, which is used for **transforming XML _data_**. The modern perspective and understanding is therefore not on "XML as a markup language for documents which are presented to a web browser, or converted to a form suitable for printing, such as PDF or PostScript", but, in general terms, on "XML as a means to represent (highly-structured) data, which can be arbitrarily transformed by XSLT". _If_ the aspect of formatting semantics is relevant to us, then we'd need to consider and describe the XSL Formatting Objects (**XSL-FO**) vocabulary. This is beyond the scope of this document. Our focus is on the transformation of XML data via XSL transformations. @@ -51,12 +54,14 @@ The **XSL transform** turns the so-called **source tree** into a **result tree** A minimal example of the (1) XML input data, (2) a corresponding XSL transformation and the (3) generated output is the following: -##### XML data (`.xml` file): +##### XML data (`.xml` file) + ```xml 1984 ``` -##### XSL stylesheet (`.xslt` file): +##### XSL stylesheet (`.xslt` file) + ```xml @@ -66,11 +71,13 @@ A minimal example of the (1) XML input data, (2) a corresponding XSL transformat ``` ##### Output: (`.html` file) + ```html

1984

``` In this example: + 1. The **XML** holds the input information (`1984`). 2. The **XSL stylesheet** specifies how that information should be formatted (it takes the title and places it inside HTML). 3. The **XSL transformation** processes both files to create the **final HTML result** (`

1984

`). @@ -80,11 +87,13 @@ In this example: A slightly more complex example is the following: **XML data** (`.xml` file): + ```xml Alice30 ``` **XSL stylesheet** (`.xslt` file): + ```xml @@ -94,6 +103,7 @@ A slightly more complex example is the following: ``` **Output:** (`.html` file) + ```html

Alice is 30 years old.

``` @@ -105,6 +115,7 @@ A slightly more complex example is the following: A rather straightforward but specific example could be the following _conversion from XML data to **RDF**_: **XML**: + ```xml Alice @@ -113,6 +124,7 @@ A rather straightforward but specific example could be the following _conversion ``` **XSL stylesheet**: + ```xml @@ -128,6 +140,7 @@ A rather straightforward but specific example could be the following _conversion ``` **Result (RDF/XML)**: + ```xml @@ -139,17 +152,20 @@ A rather straightforward but specific example could be the following _conversion ``` In this example: + 1. The **XSLT processor** matches the `` element in the input XML. 2. It **creates** an `` root element with the required namespaces. 3. It **constructs** a `` resource, using the `id` attribute as the subject URI. 4. It **writes** a `` element with the value from ``. 5. It **adds** a `` element with a `mailto:` URI built from ``. 6. It **outputs** the final RDF/XML document representing the RDF triples. + ##### JSON style The same straightforward example of an XML to RDF conversion, but using the `JSON-LD` style for the result: **XSL stylesheet** + ```xml @@ -165,6 +181,7 @@ The same straightforward example of an XML to RDF conversion, but using the `JSO ``` **Result (JSON-LD style)**: + ```json { "@id": "#p1", @@ -205,6 +222,7 @@ More conceptually, other use cases of XSLT include: - **Content publishing** — generating multiple output formats (HTML, PDF, EPUB) from a single XML source. - **Configuration and code generation** — producing scripts, configuration files, or documentation from XML metadata. - **Visualization** — creating SVG charts, diagrams, or interactive web elements from structured XML data. + ## How does XSL relate to RDF? **XSL**, specifically **XSLT**, relates to **RDF** by providing a way to **transform XML data into RDF serializations** (like RDF/XML or JSON-LD). It acts as a bridge between structured XML sources and Semantic Web representations, enabling automated RDF generation from existing XML-based datasets. @@ -218,7 +236,7 @@ To import XML into a **Knowledge Graph**, you typically: 3. **Validate the RDF** — check it against the ontology or SHACL shapes. 4. **Load the RDF** — import it into a **triple store** or **graph database** (e.g., Fuseki, GraphDB, Neo4j). -Whereas this process *does* the job of importing XML into a knowledge graph, using a **data integration** solution based on **knowledge graphs**, such as [eccenca Corporate Memory](https://eccenca.com/products/enterprise-knowledge-graph-platform-corporate-memory) (CMEM), is a *much better fit*. For an example on this, see the [tutorial on lifting data from an XML source](https://documentation.eccenca.com/latest/build/lift-data-from-json-and-xml-sources/), and notice how each of the steps (mapping, transforming, validating, loading) is realized. In such an improved setting, notice how the `xsltOperator` plugin is _**not** used for transforming XML into RDF_, but only for _transforming the data you want to import and bring it to the XML format_. The second step of the list (_transform the XML into RDF_) is taken care of by CMEM itself. The usage of XSLT is, therefore, limited to what is required by your input data and data processing requirements, not by the technicalities behind the semantic data integration. +Whereas this process _does_ the job of importing XML into a knowledge graph, using a **data integration** solution based on **knowledge graphs**, such as [eccenca Corporate Memory](https://eccenca.com/products/enterprise-knowledge-graph-platform-corporate-memory) (CMEM), is a _much better fit_. For an example on this, see the [tutorial on lifting data from an XML source](https://documentation.eccenca.com/latest/build/lift-data-from-json-and-xml-sources/), and notice how each of the steps (mapping, transforming, validating, loading) is realized. In such an improved setting, notice how the `xsltOperator` plugin is _**not** used for transforming XML into RDF_, but only for _transforming the data you want to import and bring it to the XML format_. The second step of the list (_transform the XML into RDF_) is taken care of by CMEM itself. The usage of XSLT is, therefore, limited to what is required by your input data and data processing requirements, not by the technicalities behind the semantic data integration. Notice as well that the source of XML data does not need to be an XML _file_. An alternative could be a Web API providing XML instead of JSON responses. See the [tutorial on extracting data from a Web API](https://documentation.eccenca.com/latest/build/extracting-data-from-a-web-api/), and use an **XML parser** and **XML Dataset** instead of the JSON variants described in the tutorial. Otherwise, the process is the same. @@ -239,4 +257,4 @@ The XSLT file to be used for transforming XML. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/dataset/Hive.md b/docs/build/reference/dataset/Hive.md index 9180da3d..5bda43c0 100644 --- a/docs/build/reference/dataset/Hive.md +++ b/docs/build/reference/dataset/Hive.md @@ -2,10 +2,12 @@ title: "Hive database" description: "Read from or write to an embedded Apache Hive endpoint." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Hive database + @@ -131,4 +133,4 @@ The source internal encoding, e.g., UTF8, ISO-8859-1 ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/dataset/Jdbc.md b/docs/build/reference/dataset/Jdbc.md index a276c8eb..49518c41 100644 --- a/docs/build/reference/dataset/Jdbc.md +++ b/docs/build/reference/dataset/Jdbc.md @@ -2,10 +2,12 @@ title: "Remote SQL endpoint" description: "Connect to an existing JDBC endpoint." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Remote SQL endpoint + @@ -31,7 +33,8 @@ Most of the dataset parameters are passed directly to the driver. Please make sure that you use the correct syntax for each DBMS, otherwise you may get unintuitive errors. Here are templates for supported database systems: -``` + +```text oracle (external driver needed): jdbc:oracle:thin:@{host}[:{port}]/{database} @@ -81,14 +84,15 @@ For some DBMS and specific JDBC dataset configurations we support these optimize Supported DBMS: - MySQL and MariaDB (full support for versions 8.0.19+ and 10.4+, resp.): - - if older DBMS versions are used some dataset options like 'groupBy' might not be supported but equivalent queries will - - the same is true when older driver jars then the one provided by eccenca are used - - both use the MariaDB JDBC driver - - uses `LOAD DATA LOCAL INFILE` internally - - only applies when appending data to an existing table and having `Force Spark Execution` disabled - - Both the server parameter `local_infile` and the client parameter `allowLoadLocalInfile` must be enabled, e.g. by adding `allowLoadLocalInfile=true` to the JDBC URL. + - if older DBMS versions are used some dataset options like 'groupBy' might not be supported but equivalent queries will + - the same is true when older driver jars then the one provided by eccenca are used + - both use the MariaDB JDBC driver + - uses `LOAD DATA LOCAL INFILE` internally + - only applies when appending data to an existing table and having `Force Spark Execution` disabled + - Both the server parameter `local_infile` and the client parameter `allowLoadLocalInfile` must be enabled, e.g. by adding `allowLoadLocalInfile=true` to the JDBC URL. For MySQL starting with version 8 the `local_infile` parameter is by default disabled! - - If during writing to a MySQL/MariaDB a `[…] You have an error in your SQL syntax […]` error is encountered make sure ANSIquotes are used. + + - If during writing to a MySQL/MariaDB a `[…] You have an error in your SQL syntax […]` error is encountered make sure ANSIquotes are used. `sql_mode=ANSI_QUOTES` can be set via a URL parameter to the JDBC connection string like: ```sh @@ -116,7 +120,7 @@ spark.sql.options { # jdbc:db2://host:port) is used to specify the driver. For each protocol on the list a jar classname and optional download # location can be provided. jdbc.drivers = "db2,mysql" - + # Some database systems use licenses that are to loose or restrictive for us to ship the drivers. Therefore a path # to a jar file containing the driver and the name of driver can be specified here. jdbc.db2.jar = "/home/user/Jars/db2jcc-db2jcc4.jar" @@ -132,7 +136,7 @@ spark.sql.options { In general, it will not work to upgrade a JDBC driver by providing an external driver for a database that is already packaged with eccenca Dataintegration. -The driver delivered with eccenca Dataintegration will be preferred. Driver names (configured via e.g. `spark.sql.options.jdbc.drivers = "mssql"`) will be ignored if JDBC URLs starting with, in this example `jdbc:mssql...` , are already supported in the dataset. +The driver delivered with eccenca Dataintegration will be preferred. Driver names (configured via e.g. `spark.sql.options.jdbc.drivers = "mssql"`) will be ignored if JDBC URLs starting with, in this example `jdbc:mssql...` , are already supported in the dataset. ### Recommended DBMS versions @@ -294,7 +298,7 @@ URL for retrieving tokens, when using MS SQL Active Directory token based authen ### Service principal name (Azure Active Directory) -Service Principal Name identifying the resource. Usually a static URL like https://database.windows.net. +Service Principal Name identifying the resource. Usually a static URL like . - ID: `spnName` - Datatype: `string` @@ -360,5 +364,3 @@ If set to true, Spark will be used for querying the database, even if the local - Datatype: `boolean` - Default Value: `false` - - diff --git a/docs/build/reference/dataset/LocalInternalDataset.md b/docs/build/reference/dataset/LocalInternalDataset.md index 4bd41451..9f0b070c 100644 --- a/docs/build/reference/dataset/LocalInternalDataset.md +++ b/docs/build/reference/dataset/LocalInternalDataset.md @@ -2,10 +2,12 @@ title: "Internal dataset (single graph)" description: "Dataset for storing entities between workflow steps. This variant does use the same graph for all internal datasets in a workflow. The underlying dataset type can be configured using the `dataset.internal.*` configuration parameters." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Internal dataset (single graph) + @@ -19,4 +21,4 @@ Dataset for storing entities between workflow steps. This variant does use the s ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/dataset/SnowflakeJdbc.md b/docs/build/reference/dataset/SnowflakeJdbc.md index 83e5c19c..fee6ca53 100644 --- a/docs/build/reference/dataset/SnowflakeJdbc.md +++ b/docs/build/reference/dataset/SnowflakeJdbc.md @@ -2,10 +2,12 @@ title: "Snowflake SQL endpoint" description: "Connect to Snowflake JDBC endpoint." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Snowflake SQL endpoint + @@ -274,4 +276,3 @@ Optional pause between queries in ms. - Default Value: `2000` - diff --git a/docs/build/reference/dataset/alignment.md b/docs/build/reference/dataset/alignment.md index be29102a..d9e01539 100644 --- a/docs/build/reference/dataset/alignment.md +++ b/docs/build/reference/dataset/alignment.md @@ -2,10 +2,12 @@ title: "Alignment" description: "Writes the alignment format specified at http://alignapi.gforge.inria.fr/format.html." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Alignment + @@ -15,6 +17,7 @@ tags: The Alignment plugin provides support for **writing alignment files**, following the format specified by the Alignment API ([format specification](https://moex.gitlabpages.inria.fr/alignapi/format.html)) and described in [Semantic Web Journal, SWJ60](https://www.semantic-web-journal.net/sites/default/files/swj60_1.pdf). It is designed to integrate seamlessly with Silk and CMEM BUILD's **modular, task-specific dataset architecture**, allowing users to produce alignment links in a structured and standardized way. This plugin **focuses exclusively on writing links** between entities. Unlike other datasets, it does **not provide reading of entities**, transformations, or additional processing. Its purpose is simple and clear: to **export alignment links in a format compatible with alignment-aware tools or downstream processing**. + ## General Principles The Alignment plugin reflects two guiding principles: @@ -90,4 +93,4 @@ The alignment file. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/dataset/avro.md b/docs/build/reference/dataset/avro.md index 04317229..56d24534 100644 --- a/docs/build/reference/dataset/avro.md +++ b/docs/build/reference/dataset/avro.md @@ -2,10 +2,12 @@ title: "Avro" description: "Read from or write to an Apache Avro file." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Avro + @@ -15,6 +17,7 @@ tags: The Avro dataset plugin in BUILD provides the ability to read from or write to Apache Avro files. It is a **Spark-optimized dataset**, designed to leverage Spark’s in-memory, parallel execution model and take advantage of Avro-specific optimizations. Avro supports **schema evolution** and **efficient serialization**, enabling Spark to process structured data reliably across workflow changes. While columnar optimizations like Parquet or ORC are not applicable, Avro provides **fast row-based access** and compact storage, making it suitable for workloads where the data structure may change over time or full-row access is common. + ### Key Features - **Row-based storage format**: Avro stores data row-wise, enabling efficient serialization and deserialization for distributed processing. @@ -39,6 +42,7 @@ This dataset could be stored in a file `transactions.avro` and used in workflows For more information on the Avro format and its optimizations, see the [Apache Avro project page](https://avro.apache.org/). --- + ### Comparison of Spark-optimized datasets The following table summarizes the key differences and typical use cases of the main Spark-optimized datasets supported in CMEM BUILD. It provides a quick reference for understanding the optimizations, storage formats, and workflow suitability for ORC, Parquet, and Avro datasets. @@ -102,4 +106,4 @@ The file encoding, e.g., UTF8, ISO-8859-1 ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/dataset/binaryFile.md b/docs/build/reference/dataset/binaryFile.md index 30b203f6..94502e92 100644 --- a/docs/build/reference/dataset/binaryFile.md +++ b/docs/build/reference/dataset/binaryFile.md @@ -2,10 +2,12 @@ title: "Binary file" description: "Reads and writes binary files. A typical use-case for this dataset is to process PDF documents or images." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Binary file + @@ -15,6 +17,7 @@ Reads and writes binary files. A typical use-case for this dataset is to process ## ZIP files This dataset can be used to compress/decompress ZIP files. If a ZIP file is configured, the behaviour is as follows: + - Writing a ZIP file to this dataset will overwrite the configured ZIP file. - Writing one or many non-ZIP files will overwrite the dataset file with a ZIP that contains all written files. - When reading files, the dataset will return all files inside the ZIP that match the configured regex. If the regex is empty, the ZIP file itself will be returned. @@ -55,4 +58,4 @@ If the file is a ZIP file, read files are filtered via this regex. If empty, the ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/dataset/csv.md b/docs/build/reference/dataset/csv.md index a8528428..ae8d13bb 100644 --- a/docs/build/reference/dataset/csv.md +++ b/docs/build/reference/dataset/csv.md @@ -2,10 +2,12 @@ title: "CSV" description: "Read from or write to an CSV file." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # CSV + @@ -159,7 +161,7 @@ This is deprecated, use the 'Clear dataset' operator instead to clear a dataset -### Trim whitespace and non-printable characters. +### Trim whitespace and non-printable characters If set to true, this will trim whitespace and non-printable characters from the contents of the CSV dataset. @@ -168,4 +170,3 @@ If set to true, this will trim whitespace and non-printable characters from the - Default Value: `false` - diff --git a/docs/build/reference/dataset/eccencaDataPlatform.md b/docs/build/reference/dataset/eccencaDataPlatform.md index 9fadedef..50972209 100644 --- a/docs/build/reference/dataset/eccencaDataPlatform.md +++ b/docs/build/reference/dataset/eccencaDataPlatform.md @@ -2,10 +2,12 @@ title: "Knowledge Graph" description: "Read RDF from or write RDF to a Knowledge Graph embedded in Corporate Memory." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Knowledge Graph + @@ -26,8 +28,8 @@ The plugin itself is part of [**eccenca Build**](https://documentation.eccenca.c following aspect (see the [Getting Started](https://documentation.eccenca.com/latest/getting-started/) and the remaining documentation for further details): -* **Explore** is for browsing and exploring knowledge graphs -* **Build** is for creating and integrating knowledge graphs +- **Explore** is for browsing and exploring knowledge graphs +- **Build** is for creating and integrating knowledge graphs This plugin is thus responsible for _creating_ and _integrating_ knowledge graphs within eccenca Build. The knowledge graph itself is part of `DataPlatform`, which can be @@ -61,7 +63,7 @@ The URI of the named graph. ### SPARQL query timeout (ms) -SPARQL query timeout in milliseconds. By default, a value of zero is used. This zero value has a symbolic character: it means that the timeout of SPARQL select and update queries is configured via the properties `silk.remoteSparqlEndpoint.defaults.connection.timeout.ms and `silk.remoteSparqlEndpoint.defaults.read.timeout.ms` for the default connection and read timeouts. To overwrite these configured values, specify a (common) timeout greater than zero milliseconds. +SPARQL query timeout in milliseconds. By default, a value of zero is used. This zero value has a symbolic character: it means that the timeout of SPARQL select and update queries is configured via the properties `silk.remoteSparqlEndpoint.defaults.connection.timeout.ms and`silk.remoteSparqlEndpoint.defaults.read.timeout.ms` for the default connection and read timeouts. To overwrite these configured values, specify a (common) timeout greater than zero milliseconds. - ID: `sparqlTimeout` - Datatype: `int` @@ -172,4 +174,3 @@ Use streaming HTTP upload (chunked transfer encoding) instead of multipart file - Default Value: `false` - diff --git a/docs/build/reference/dataset/excel.md b/docs/build/reference/dataset/excel.md index 905b70f6..23248a3b 100644 --- a/docs/build/reference/dataset/excel.md +++ b/docs/build/reference/dataset/excel.md @@ -2,10 +2,12 @@ title: "Excel" description: "Read from or write to an Excel workbook in Open XML format (XLSX). The sheet is selected by specifying it as type in the subsequent workflow operator." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Excel + @@ -68,4 +70,3 @@ Streaming enables reading and writing large Excels files. Warning: Be careful to - Default Value: `true` - diff --git a/docs/build/reference/dataset/file.md b/docs/build/reference/dataset/file.md index 6fe5e2b1..6a27d00f 100644 --- a/docs/build/reference/dataset/file.md +++ b/docs/build/reference/dataset/file.md @@ -2,10 +2,12 @@ title: "RDF file" description: "Dataset which retrieves and writes all entities from/to an RDF file. For reading, the dataset is loaded in-memory and thus the size is restricted by the available memory. Large datasets should be loaded into an external RDF store and retrieved using the SPARQL dataset instead." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # RDF file + @@ -157,6 +159,7 @@ When the dataset is used (e.g. queried or read by downstream components), it beh - **Graph:** (empty → default graph) Use case: + - Small RDF dataset loaded from a Turtle file for exploration and transformations. ### 6.2 N-Quads file with explicit graph @@ -166,6 +169,7 @@ Use case: - **Graph:** `http://example.com/graph/main` Use case: + - Large export containing multiple graphs, but only a specific named graph is used in the project. ### 6.3 ZIP archive with multiple RDF files @@ -175,6 +179,7 @@ Use case: - **ZIP file regex:** `.*\.ttl` Use case: + - ZIP archive containing various files; only Turtle files are loaded into the dataset. ## 7. When to use this dataset @@ -246,4 +251,3 @@ If the input resource is a ZIP file, files inside the file are filtered via this - Default Value: `.*` - diff --git a/docs/build/reference/dataset/googlespreadsheet.md b/docs/build/reference/dataset/googlespreadsheet.md index f64cdbd6..8c212394 100644 --- a/docs/build/reference/dataset/googlespreadsheet.md +++ b/docs/build/reference/dataset/googlespreadsheet.md @@ -2,10 +2,12 @@ title: "Excel (Google Drive)" description: "Read data from a remote Google Spreadsheet." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Excel (Google Drive) + @@ -16,7 +18,7 @@ It will automatically correct the URL and add the "export as xlsx" option to a n that will be used to download an Excel Spreadsheet. The download will be cached and treated the same way as an xlsx file in the Excel Dataset. -### Caching +## Caching The advanced parameter `invalidateCacheAfter` allows the user to specify a duration of the file cache after which it is refreshed. @@ -69,5 +71,3 @@ Duration until file based cache is invalidated. - Datatype: `duration` - Default Value: `PT5M` - - diff --git a/docs/build/reference/dataset/inMemory.md b/docs/build/reference/dataset/inMemory.md index a5c9a541..7e65d330 100644 --- a/docs/build/reference/dataset/inMemory.md +++ b/docs/build/reference/dataset/inMemory.md @@ -2,10 +2,12 @@ title: "In-memory dataset" description: "A Dataset that holds all data in-memory." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # In-memory dataset + @@ -15,6 +17,7 @@ tags: The **in-memory dataset** is a small embedded RDF store that keeps all data **in memory** and exposes it via SPARQL. It is intended as a **temporary working graph** inside workflows, not as a large or persistent storage. Typical use cases: + - Collecting intermediate results during a workflow run. - Storing small lookup graphs used by downstream operators. - Testing or prototyping workflows without configuring an external RDF store. @@ -28,6 +31,7 @@ Typical use cases: - After an application restart, the dataset contents are empty again. Within a workflow: + - The dataset can be used as both **input** and **output**: - Upstream operators can write triples/entities/links into it. - Downstream operators can read from it via SPARQL-based mechanisms. @@ -119,4 +123,3 @@ This is deprecated, use the 'Clear dataset' operator instead to clear a dataset - Default Value: `false` - diff --git a/docs/build/reference/dataset/index.md b/docs/build/reference/dataset/index.md index 6620a922..8e8cd9af 100644 --- a/docs/build/reference/dataset/index.md +++ b/docs/build/reference/dataset/index.md @@ -5,7 +5,9 @@ tags: - Build - Reference --- + # Datasets + Datasets are collections of data that can be read or written. @@ -14,7 +16,7 @@ Datasets are collections of data that can be read or written. | Name | Description | |-------------:|:-------------------------| - | [Alignment](alignment.md) | Writes the alignment format specified at http://alignapi.gforge.inria.fr/format.html. | + | [Alignment](alignment.md) | Writes the alignment format specified at . | | [Avro](avro.md) | Read from or write to an Apache Avro file. | | [Binary file](binaryFile.md) | Reads and writes binary files. A typical use-case for this dataset is to process PDF documents or images. | | [CSV](csv.md) | Read from or write to an CSV file. | diff --git a/docs/build/reference/dataset/internal.md b/docs/build/reference/dataset/internal.md index e7fa4689..b8ec25aa 100644 --- a/docs/build/reference/dataset/internal.md +++ b/docs/build/reference/dataset/internal.md @@ -2,10 +2,12 @@ title: "Internal dataset" description: "Dataset for storing entities between workflow steps. The underlying dataset type can be configured using the `dataset.internal.*` configuration parameters." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Internal dataset + @@ -29,4 +31,4 @@ The RDF graph that is used for storing internal data ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/dataset/json.md b/docs/build/reference/dataset/json.md index 1bd227f4..c5e22640 100644 --- a/docs/build/reference/dataset/json.md +++ b/docs/build/reference/dataset/json.md @@ -2,10 +2,12 @@ title: "JSON" description: "Read from or write to a JSON or JSON Lines file." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # JSON + @@ -17,6 +19,7 @@ Typically, this dataset is used to transform an JSON file to another format, e.g In addition to plain JSON files, *JSON Lines* files can also be read. For reading, the JSON dataset supports a number of special paths: + - `#id` Is a special syntax for generating an id for a selected element. It can be used in URI patterns for entities which do not provide an identifier. Examples: `http://example.org/{#id}` or `http://example.org/{/pathToEntity/#id}`. - `#text` retrieves the text of the selected node. - The backslash can be used to navigate to the parent JSON node, e.g., `\parent/key`. The name of the backslash key (here `parent`) is ignored. @@ -74,7 +77,7 @@ The path to the elements to be read, starting from the root element, e.g., '/Per ### URI pattern (deprecated) -A URI pattern, e.g., http://namespace.org/{ID}, where {path} may contain relative paths to elements +A URI pattern, e.g., , where {path} may contain relative paths to elements - ID: `uriPattern` - Datatype: `string` @@ -111,4 +114,3 @@ If the input resource is a ZIP file, files inside the file are filtered via this - Default Value: `^(?!.*[\/\\]\..*$|^\..*$).*\.jsonl?$` - diff --git a/docs/build/reference/dataset/multiCsv.md b/docs/build/reference/dataset/multiCsv.md index fa764808..6d6d1830 100644 --- a/docs/build/reference/dataset/multiCsv.md +++ b/docs/build/reference/dataset/multiCsv.md @@ -2,10 +2,12 @@ title: "Multi CSV ZIP" description: "Reads from or writes to multiple CSV files from/to a single ZIP file." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Multi CSV ZIP + @@ -139,7 +141,7 @@ This is deprecated, use the 'Clear dataset' operator instead to clear a dataset -### Optionally trim whitespace and non-printable characters. +### Optionally trim whitespace and non-printable characters If set to true, this will trim whitespace and non-printable characters from the contents of the CSV dataset. @@ -148,4 +150,3 @@ If set to true, this will trim whitespace and non-printable characters from the - Default Value: `false` - diff --git a/docs/build/reference/dataset/neo4j.md b/docs/build/reference/dataset/neo4j.md index c0b5c523..5aa37ea9 100644 --- a/docs/build/reference/dataset/neo4j.md +++ b/docs/build/reference/dataset/neo4j.md @@ -2,10 +2,12 @@ title: "Neo4j" description: "Neo4j graph" icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Neo4j + @@ -15,7 +17,7 @@ Supports reading and writing Neo4j graphs. The following sections outline how gr For more information about Neo4j, please refer to the [Neo4j documentation](https://neo4j.com/docs/). -### Nodes +## Nodes For each entity that is written to a Neo4j dataset, a _node_ will be created. A property `uri` will be added to each generated node, which holds the URI of the original entity. @@ -52,6 +54,7 @@ In eccenca DataIntegration, URIs are typically used to uniquely identify classes While URIs are central in RDF, Neo4j does allow arbitrary names and does not have any special support for URIs. When generating Neo4j labels, properties and relationships, URIs will be shortened according to the following rules. + - If a registered project prefix matches a URI, a name `{prefixName}_{localPart}` will be generated. For instance, `http://xmlns.com/foaf/0.1/name` will become `foaf_name`. Note that underscores (`_`) are used instead of colons (`:`) to separate the namespace and the local name. The reason is that colons are reserved in the Cypher query language and some tools don't escape properly and fail on databases that use colons in names. @@ -138,5 +141,3 @@ This is deprecated, use the 'Clear dataset' operator instead to clear a dataset - Datatype: `boolean` - Default Value: `false` - - diff --git a/docs/build/reference/dataset/office365preadsheet.md b/docs/build/reference/dataset/office365preadsheet.md index bbfa5bdf..950e0a4a 100644 --- a/docs/build/reference/dataset/office365preadsheet.md +++ b/docs/build/reference/dataset/office365preadsheet.md @@ -2,10 +2,12 @@ title: "Excel (OneDrive, Office365)" description: "Read data from a remote onedrive or Office365 Spreadsheet." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Excel (OneDrive, Office365) + @@ -22,9 +24,9 @@ Onedrive links look like `https://1drv.ms/x/s!AucULvzmJ-dsdfsfgaIcyWP_XY_G4w?e=y Onedrive (based one sharepoint, for businesses) links look like `https://eccencagmbh-my.sharepoint.com/:x:/g/personal/person_eccenca_com/EdEMTEw1dclHiEZXyvy8P4YBit8wSyGsiwU5Kt__sQOZzw` The first type should always work is not recommended for this dataset. The second type requires to set up an application in Microsoft EntraID (formerly Azure Active Directory). -EntraID: https://docs.microsoft.com/azure/active-directory/develop/v2-overview +EntraID: Instructions and examples can be found here: -https://github.com/Azure-Samples/ms-identity-msal-java-samples/tree/main/3-java-servlet-web-app/1-Authentication/sign-in + After following the steps access to sharepoint/onedrive for business can be setup in the application.conf file for eccenca DataIntegration. @@ -94,4 +96,3 @@ Duration until file based cache is invalidated. - Default Value: `PT5M` - diff --git a/docs/build/reference/dataset/orc.md b/docs/build/reference/dataset/orc.md index de2d9d00..e1e7addc 100644 --- a/docs/build/reference/dataset/orc.md +++ b/docs/build/reference/dataset/orc.md @@ -2,10 +2,12 @@ title: "ORC" description: "Read from or write to an Apache ORC file." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # ORC + @@ -15,12 +17,14 @@ tags: The ORC dataset plugin in BUILD provides the ability to read from or write to Apache ORC files. It is a **Spark-optimized dataset**, designed to leverage Spark’s in-memory, parallel execution model and take advantage of ORC-specific optimizations. ORC supports optimizations such as **column pruning** and **predicate pushdown**, which allow queries to efficiently read only the needed data. This makes ORC particularly suitable for workflows where only a subset of columns or filtered rows are required, improving performance for large-scale transformations. + ### Key Features - **Columnar storage format**: ORC stores data in a column-oriented manner, enabling efficient compression and retrieval of individual columns. - **Partitioning**: Optional support for partitioned outputs, allowing Spark to parallelize data processing effectively. - **Compression**: Supports multiple compression algorithms, e.g., Snappy or Zlib, reducing storage footprint and improving I/O performance. - **Schema handling**: Supports explicit or inferred schemas, with the schema applied to the dataset as a whole, not individual rows. + ### Example An ORC dataset in CMEM Build might look conceptually like this: @@ -38,6 +42,7 @@ This dataset could be stored in a file `transactions.orc` and used in workflows For more information on the ORC format and its optimizations, see the [Apache ORC project page](https://orc.apache.org/). --- + ### Comparison of Spark-optimized datasets The following table summarizes the key differences and typical use cases of the main Spark-optimized datasets supported in CMEM BUILD. It provides a quick reference for understanding the optimizations, storage formats, and workflow suitability for ORC, Parquet, and Avro datasets. @@ -121,4 +126,4 @@ The file encoding, e.g., UTF8, ISO-8859-1 ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/dataset/parquet.md b/docs/build/reference/dataset/parquet.md index 90024582..d7a1fae8 100644 --- a/docs/build/reference/dataset/parquet.md +++ b/docs/build/reference/dataset/parquet.md @@ -2,10 +2,12 @@ title: "Parquet" description: "Read from or write to an Apache Parquet file." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Parquet + @@ -15,12 +17,14 @@ tags: The Parquet dataset plugin in BUILD provides the ability to read from or write to Apache Parquet files. It is a **Spark-optimized dataset**, designed to leverage Spark’s in-memory, parallel execution model and take advantage of Parquet-specific optimizations. Parquet supports optimizations such as **column pruning** and **predicate pushdown**, which enable Spark to read only the required columns or rows. This makes Parquet particularly effective for workflows where selective access to data is needed, improving performance for large-scale transformations. + ### Key Features - **Columnar storage format**: Parquet stores data in a column-oriented manner, enabling efficient compression and retrieval of individual columns. - **Partitioning**: Optional support for partitioned outputs, allowing Spark to parallelize data processing effectively. - **Compression**: Supports multiple compression algorithms, e.g., Snappy, Gzip, or LZO, reducing storage footprint and improving I/O performance. - **Schema handling**: Supports explicit or inferred schemas, with the schema applied to the dataset as a whole, not individual rows. + ### Example A Parquet dataset in CMEM Build might look conceptually like this: @@ -32,11 +36,13 @@ A Parquet dataset in CMEM Build might look conceptually like this: | 3 | 1003 | 75.25 | 2025-11-01 11:20 | This dataset could be stored in a file `transactions.parquet` and used in workflows where only certain columns or filtered rows are needed. Spark can read only the relevant columns or rows matching a condition, thanks to Parquet’s **column pruning** and **predicate pushdown** optimizations, making transformations efficient even at large scale. + ### Reference For more information on the Parquet format and its optimizations, see the [Apache Parquet project page](https://parquet.apache.org/). --- + ### Comparison of Spark-optimized datasets The following table summarizes the key differences and typical use cases of the main Spark-optimized datasets supported in CMEM BUILD. It provides a quick reference for understanding the optimizations, storage formats, and workflow suitability for ORC, Parquet, and Avro datasets. @@ -120,4 +126,4 @@ The file encoding, e.g., UTF8, ISO-8859-1 ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/dataset/sparkView.md b/docs/build/reference/dataset/sparkView.md index 4288af73..f4b9cf04 100644 --- a/docs/build/reference/dataset/sparkView.md +++ b/docs/build/reference/dataset/sparkView.md @@ -2,10 +2,12 @@ title: "Embedded Spark SQL view" description: "Deprecated: Use the embedded SQL endpoint dataset instead." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Embedded Spark SQL view + @@ -99,4 +101,4 @@ If true, basic types will be used for types that otherwise would result in clien ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/dataset/sparqlEndpoint.md b/docs/build/reference/dataset/sparqlEndpoint.md index 0e140998..8201b7c3 100644 --- a/docs/build/reference/dataset/sparqlEndpoint.md +++ b/docs/build/reference/dataset/sparqlEndpoint.md @@ -2,10 +2,12 @@ title: "SPARQL endpoint" description: "Connects to an existing SPARQL endpoint." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # SPARQL endpoint + @@ -31,7 +33,7 @@ queries. Additionally, it can execute ## Example usage A very simple example showcasing the usage of this plugin is the following idea: Use an online SPARQL Query Editor such -as https://dbpedia.org/sparql, with a simple SPARQL query like `select distinct ?Concept where {[] a ?Concept} LIMIT 10` +as , with a simple SPARQL query like `select distinct ?Concept where {[] a ?Concept} LIMIT 10` or similar. Use this plugin as a source dataset, and transform or transfer the SPARQL query results into a sink dataset such as a CSV file. A similar or related showcase example involves considering other output datasets such as an in-memory dataset or a Knowledge Graph such as the one handled by the Knowledge Graph plugin, which is the flagship RDF @@ -102,7 +104,7 @@ Enforces the correct ordering of values, if set to `true` (default). ### SPARQL query timeout (ms) -SPARQL query timeout in milliseconds. By default, a value of zero is used. This zero value has a symbolic character: it means that the timeout of SPARQL select and update queries is configured via the properties `silk.remoteSparqlEndpoint.defaults.connection.timeout.ms and `silk.remoteSparqlEndpoint.defaults.read.timeout.ms` for the default connection and read timeouts. To overwrite these configured values, specify a (common) timeout greater than zero milliseconds. +SPARQL query timeout in milliseconds. By default, a value of zero is used. This zero value has a symbolic character: it means that the timeout of SPARQL select and update queries is configured via the properties `silk.remoteSparqlEndpoint.defaults.connection.timeout.ms and`silk.remoteSparqlEndpoint.defaults.read.timeout.ms` for the default connection and read timeouts. To overwrite these configured values, specify a (common) timeout greater than zero milliseconds. - ID: `sparqlTimeout` - Datatype: `int` @@ -183,4 +185,3 @@ This is deprecated, use the 'Clear dataset' operator instead to clear a dataset - Default Value: `false` - diff --git a/docs/build/reference/dataset/sqlEndpoint.md b/docs/build/reference/dataset/sqlEndpoint.md index dec05097..83016c9f 100644 --- a/docs/build/reference/dataset/sqlEndpoint.md +++ b/docs/build/reference/dataset/sqlEndpoint.md @@ -2,10 +2,12 @@ title: "Embedded SQL endpoint" description: "Provides a JDBC endpoint that exposes workflow or transformation results as tables, which can be queried using SQL." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Embedded SQL endpoint + @@ -14,13 +16,14 @@ tags: Within BUILD, several Spark-aware datasets exist — Avro, Parquet, ORC, HDFS, Hive — each optimized to leverage Spark’s parallel, in-memory execution. Complementing these table-based and file-based datasets, the **Embedded SQL Endpoint** provides a **queryable, relational interface** to workflow results or transformations. Unlike file-based formats, this endpoint exposes results as **virtual tables**, accessible via **JDBC or other SQL clients**, enabling downstream queries, reporting, or integration without requiring persistent storage. -### Concept +### Concept The Embedded SQL Endpoint represents **workflow-generated tables**. It is **not a general-purpose query endpoint**; each table corresponds directly to workflow outputs and serves as a **source or sink** within the workflow, functioning as a **dataset** like other Spark-aware datasets and integrating with BUILD plugins in the usual way. The endpoint allows workflows to **publish results as tables** in a format compatible with standard SQL tooling. Multiple tables can be exposed per workflow if complex mappings exist. Table names can be **customized using a prefix** or automatically generated for convenience. By caching tables in memory, Spark ensures that queries execute efficiently while maintaining fault tolerance and parallelism inherent in the Spark ecosystem. Conceptually, these tables can be **consumed by other workflows or plugins** without needing to manage Spark execution details. + ### Key Features - **Virtual table exposure**: Any workflow or transformation result can be turned into a SQL-accessible table. @@ -126,4 +129,4 @@ Mapping of column names. Similar to aliases E.g. 'c1:c2' would rename column c1 ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/dataset/text.md b/docs/build/reference/dataset/text.md index ed034bce..d0c6b810 100644 --- a/docs/build/reference/dataset/text.md +++ b/docs/build/reference/dataset/text.md @@ -2,10 +2,12 @@ title: "Text" description: "Reads and writes plain text files." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # Text + @@ -14,7 +16,7 @@ Reads and writes plain text files. ## Writing -All values of each entity will be written as plain text. Multiple values per entity are separated by spaces. Each entity will be written to a new line. +All values of each entity will be written as plain text. Multiple values per entity are separated by spaces. Each entity will be written to a new line. ## Reading @@ -76,4 +78,3 @@ If the input resource is a ZIP file, files inside the file are filtered via this - Default Value: `.*` - diff --git a/docs/build/reference/dataset/xml.md b/docs/build/reference/dataset/xml.md index df544b1b..c169757b 100644 --- a/docs/build/reference/dataset/xml.md +++ b/docs/build/reference/dataset/xml.md @@ -2,10 +2,12 @@ title: "XML" description: "Read from or write to an XML file." icon: octicons/cross-reference-24 -tags: +tags: - Dataset --- + # XML + @@ -84,7 +86,7 @@ The base path when writing XML. For instance: /RootElement/Entity. Should no lon ### URI pattern -A URI pattern, e.g., http://namespace.org/{ID}, where {path} may contain relative paths to elements +A URI pattern, e.g., , where {path} may contain relative paths to elements - ID: `uriPattern` - Datatype: `string` @@ -131,4 +133,3 @@ If the input resource is a ZIP file, files inside the file are filtered via this - Default Value: `^(?!.*[\/\\]\..*$|^\..*$).*\.xml$` - diff --git a/docs/build/reference/distancemeasure/PhysicalQuantitiesDistance.md b/docs/build/reference/distancemeasure/PhysicalQuantitiesDistance.md index ee37a94e..17e8f3f8 100644 --- a/docs/build/reference/distancemeasure/PhysicalQuantitiesDistance.md +++ b/docs/build/reference/distancemeasure/PhysicalQuantitiesDistance.md @@ -2,10 +2,12 @@ title: "Compare physical quantities" description: "Computes the distance between two physical quantities." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Compare physical quantities + @@ -22,6 +24,7 @@ Comparing incompatible units will yield a validation error. Time is expressed in seconds (symbol: `s`). The following alternative symbols are supported: + * `mo_s`: day*29.53059 * `mo_g`: year/12.0 * `a`: day*365.25 @@ -39,6 +42,7 @@ The following alternative symbols are supported: Length is expressed in metres (symbol: `m`). The following alternative symbols are supported: + * `in`: c(cm*254.0) * `nmi`: m*1852.0 * `Ao`: dnm @@ -56,6 +60,7 @@ The following alternative symbols are supported: Mass is expressed in kilograms (symbol: `kg`). The following alternative symbols are supported: + * `lb`: lb * `ston`: hlb*20.0 * `t`: Mg @@ -74,6 +79,7 @@ The following alternative symbols are supported: Electric current is expressed in amperes (symbol: `A`). The following alternative symbols are supported: + * `Bi`: daA * `Gb`: cm·(A/m)*250.0/[one?] @@ -82,6 +88,7 @@ The following alternative symbols are supported: Temperature is expressed in kelvins (symbol: `K`). The following alternative symbols are supported: + * `Cel`: ℃ @@ -97,6 +104,7 @@ Luminous intensity is expressed in candelas (symbol: `cd`). Area is expressed in square metres (symbol: `m²`). The following alternative symbols are supported: + * `m2`: m² * `ar`: hm² * `syd`: ((c(cm*254.0))*12.0)*3.0² @@ -110,6 +118,7 @@ The following alternative symbols are supported: Volume is expressed in cubic metres (symbol: `㎥`). The following alternative symbols are supported: + * `st`: [㎥?] * `bf`: (c(cm*254.0)³)*144.0 * `cyd`: ((c(cm*254.0))*12.0)*3.0³ @@ -125,6 +134,7 @@ The following alternative symbols are supported: Energy is expressed in joules (symbol: `J`). The following alternative symbols are supported: + * `cal_IT`: (J*41868.0)/10000.0 * `eV`: J*1.602176487E-19 * `cal_m`: (J*419002.0)/100000.0 @@ -136,6 +146,7 @@ The following alternative symbols are supported: Angle is expressed in radians (symbol: `rad`). The following alternative symbols are supported: + * `circ`: [one?]·rad*2.0 * `gon`: ([one?]·rad/180.0)*0.9 * `deg`: [one?]·rad/180.0 @@ -145,46 +156,48 @@ The following alternative symbols are supported: ### Others -- `1/m`, derived units: `Ky`: c(1/m) -- `kg/(m·s)`, derived units: `P`: g/(s·cm) -- `bit/s`, derived units: `Bd`: bit/s -- `bit`, derived units: `By`: bit*8.0 -- `Sv` -- `N` -- `Ω`, derived units: `Ohm`: Ω -- `T`, derived units: `G`: T/10000.0 -- `sr`, derived units: `sph`: [one?]·sr*4.0 -- `F` -- `C/kg`, derived units: `R`: (C/kg)*2.58E-4 -- `cd/m²`, derived units: `sb`: cd/cm², `Lmb`: cd/([one?]·cm²) -- `Pa`, derived units: `bar`: Pa*100000.0, `atm`: Pa*101325.0 -- `kg/(m·s²)`, derived units: `att`: k(g·(m/s²)*9.80665)/cm² -- `m²/s`, derived units: `St`: cm²/s -- `A/m`, derived units: `Oe`: (A/m)*250.0/[one?] -- `kg·m²/s²`, derived units: `erg`: cm²·g/s² -- `kg/m³`, derived units: `g%`: g/dl -- `mho` -- `V` -- `lx`, derived units: `ph`: lx/10000.0 -- `m/s²`, derived units: `Gal`: cm/s², `m/s2`: m/s² -- `m/s`, derived units: `kn`: m*1852.0/h -- `m·kg/s²`, derived units: `gf`: g·(m/s²)*9.80665, `lbf`: lb·(m/s²)*9.80665, `dyn`: cm·g/s² -- `m²/s²`, derived units: `RAD`: cm²·g/(s²·hg), `REM`: cm²·g/(s²·hg) -- `C` -- `Gy` -- `Hz` -- `H` -- `lm` -- `W` -- `Wb`, derived units: `Mx`: Wb/1.0E8 -- `Bq`, derived units: `Ci`: Bq*3.7E10 -- `S` +* `1/m`, derived units: `Ky`: c(1/m) +* `kg/(m·s)`, derived units: `P`: g/(s·cm) +* `bit/s`, derived units: `Bd`: bit/s +* `bit`, derived units: `By`: bit*8.0 +* `Sv` +* `N` +* `Ω`, derived units: `Ohm`: Ω +* `T`, derived units: `G`: T/10000.0 +* `sr`, derived units: `sph`: [one?]·sr*4.0 +* `F` +* `C/kg`, derived units: `R`: (C/kg)*2.58E-4 +* `cd/m²`, derived units: `sb`: cd/cm², `Lmb`: cd/([one?]·cm²) +* `Pa`, derived units: `bar`: Pa*100000.0, `atm`: Pa*101325.0 +* `kg/(m·s²)`, derived units: `att`: k(g·(m/s²)*9.80665)/cm² +* `m²/s`, derived units: `St`: cm²/s +* `A/m`, derived units: `Oe`: (A/m)*250.0/[one?] +* `kg·m²/s²`, derived units: `erg`: cm²·g/s² +* `kg/m³`, derived units: `g%`: g/dl +* `mho` +* `V` +* `lx`, derived units: `ph`: lx/10000.0 +* `m/s²`, derived units: `Gal`: cm/s², `m/s2`: m/s² +* `m/s`, derived units: `kn`: m*1852.0/h +* `m·kg/s²`, derived units: `gf`: g·(m/s²)*9.80665, `lbf`: lb·(m/s²)*9.80665, `dyn`: cm·g/s² +* `m²/s²`, derived units: `RAD`: cm²·g/(s²·hg), `REM`: cm²·g/(s²·hg) +* `C` +* `Gy` +* `Hz` +* `H` +* `lm` +* `W` +* `Wb`, derived units: `Mx`: Wb/1.0E8 +* `Bq`, derived units: `Ci`: Bq*3.7E10 +* `S` ## Characteristics + This distance measure is not normalized, i.e., all distances start at 0 (exact match) and increase the more different the values are. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. + ## Examples **Notation:** List of values are represented via square brackets. Example: `[first, second]` represents a list of two values "first" and "second". @@ -193,8 +206,8 @@ Compares single values (as opposed to sequences of values). If multiple values a **Convert SI unit prefixes:** * Input values: - - Source: `[1 km]` - - Target: `[500 m]` + * Source: `[1 km]` + * Target: `[500 m]` * Returns: `500.0` @@ -203,8 +216,8 @@ Compares single values (as opposed to sequences of values). If multiple values a **Convert imperial and metric values:** * Input values: - - Source: `[1 km]` - - Target: `[1 mi]` + * Source: `[1 km]` + * Target: `[1 mi]` * Returns: `609.344` @@ -213,8 +226,8 @@ Compares single values (as opposed to sequences of values). If multiple values a **Validate if the compared units of measurement are compatible:** * Input values: - - Source: `[1 km]` - - Target: `[1 kg]` + * Source: `[1 km]` + * Target: `[1 kg]` * Returns: `NaN` * **Throws error:** `ValidationException` @@ -228,9 +241,9 @@ Compares single values (as opposed to sequences of values). If multiple values a The IETF BCP 47 language tag, e.g., 'en'. -- ID: `numberFormat` -- Datatype: `string` -- Default Value: `en` +* ID: `numberFormat` +* Datatype: `string` +* Default Value: `en` @@ -238,4 +251,4 @@ The IETF BCP 47 language tag, e.g., 'en'. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/cjkReadingDistance.md b/docs/build/reference/distancemeasure/cjkReadingDistance.md index aa10ccd0..219ecbf4 100644 --- a/docs/build/reference/distancemeasure/cjkReadingDistance.md +++ b/docs/build/reference/distancemeasure/cjkReadingDistance.md @@ -2,10 +2,12 @@ title: "CJK reading distance" description: "CJK Reading Distance." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # CJK reading distance + @@ -13,6 +15,7 @@ tags: CJK Reading Distance. ## Characteristics + This distance measure is not normalized, i.e., all distances start at 0 (exact match) and increase the more different the values are. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. @@ -43,4 +46,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/constantDistance.md b/docs/build/reference/distancemeasure/constantDistance.md index faceb1d5..861ad4eb 100644 --- a/docs/build/reference/distancemeasure/constantDistance.md +++ b/docs/build/reference/distancemeasure/constantDistance.md @@ -2,10 +2,12 @@ title: "Constant similarity value" description: "Always returns a constant similarity value." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Constant similarity value + @@ -13,6 +15,7 @@ tags: Always returns a constant similarity value. ## Characteristics + This distance measure is not normalized, i.e., all distances start at 0 (exact match) and increase the more different the values are. ## Parameter @@ -31,4 +34,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/cosine.md b/docs/build/reference/distancemeasure/cosine.md index 41d2d0b4..9db5c7fa 100644 --- a/docs/build/reference/distancemeasure/cosine.md +++ b/docs/build/reference/distancemeasure/cosine.md @@ -2,10 +2,12 @@ title: "Cosine" description: "Cosine Distance Measure." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Cosine + @@ -13,6 +15,7 @@ tags: Cosine Distance Measure. ## Characteristics + This distance measure is normalized, i.e., all distances are between 0 (exact match) and 1 (no similarity). Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. @@ -33,4 +36,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/date.md b/docs/build/reference/distancemeasure/date.md index 8a92a75e..eecc7f33 100644 --- a/docs/build/reference/distancemeasure/date.md +++ b/docs/build/reference/distancemeasure/date.md @@ -2,10 +2,12 @@ title: "Date" description: "The distance in days between two dates ('YYYY-MM-DD' format)." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Date + @@ -13,9 +15,11 @@ tags: The distance in days between two dates ('YYYY-MM-DD' format). ## Characteristics + This distance measure is not normalized, i.e., all distances start at 0 (exact match) and increase the more different the values are. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. + ## Examples **Notation:** List of values are represented via square brackets. Example: `[first, second]` represents a list of two values "first" and "second". @@ -24,8 +28,8 @@ Compares single values (as opposed to sequences of values). If multiple values a **Returns 0 if both dates are equal:** * Input values: - - Source: `[2003-03-01]` - - Target: `[2003-03-01]` + * Source: `[2003-03-01]` + * Target: `[2003-03-01]` * Returns: `0.0` @@ -34,8 +38,8 @@ Compares single values (as opposed to sequences of values). If multiple values a **Returns 1 if both dates are one day apart:** * Input values: - - Source: `[2003-03-01]` - - Target: `[2003-03-02]` + * Source: `[2003-03-01]` + * Target: `[2003-03-02]` * Returns: `1.0` @@ -44,8 +48,8 @@ Compares single values (as opposed to sequences of values). If multiple values a **Returns the number of days if both dates are one month apart:** * Input values: - - Source: `[2003-03-01]` - - Target: `[2003-04-01]` + * Source: `[2003-03-01]` + * Target: `[2003-04-01]` * Returns: `31.0` @@ -54,8 +58,8 @@ Compares single values (as opposed to sequences of values). If multiple values a **Returns the number of days if both dates are one year apart:** * Input values: - - Source: `[2018-03-01]` - - Target: `[2019-03-01]` + * Source: `[2018-03-01]` + * Target: `[2019-03-01]` * Returns: `365.0` @@ -64,8 +68,8 @@ Compares single values (as opposed to sequences of values). If multiple values a **Time of day is ignored:** * Input values: - - Source: `[2003-03-01]` - - Target: `[2003-03-01T06:00:00]` + * Source: `[2003-03-01]` + * Target: `[2003-03-01T06:00:00]` * Returns: `0.0` @@ -74,8 +78,8 @@ Compares single values (as opposed to sequences of values). If multiple values a **Missing days are set to 1 by default:** * Input values: - - Source: `[2003-01]` - - Target: `[2003-01-01]` + * Source: `[2003-01]` + * Target: `[2003-01-01]` * Returns: `0.0` @@ -84,8 +88,8 @@ Compares single values (as opposed to sequences of values). If multiple values a **Missing months are set to 1 by default:** * Input values: - - Source: `[2003]` - - Target: `[2003-01-01]` + * Source: `[2003]` + * Target: `[2003-01-01]` * Returns: `0.0` @@ -94,8 +98,8 @@ Compares single values (as opposed to sequences of values). If multiple values a **Missing months and days are set to 1 by default:** * Input values: - - Source: `[2018]` - - Target: `[2019]` + * Source: `[2018]` + * Target: `[2019]` * Returns: `365.0` @@ -107,8 +111,8 @@ Compares single values (as opposed to sequences of values). If multiple values a * requireMonthAndDay: `true` * Input values: - - Source: `[2003]` - - Target: `[2003-03-01]` + * Source: `[2003]` + * Target: `[2003-03-01]` * Returns: `Infinity` @@ -120,8 +124,8 @@ Compares single values (as opposed to sequences of values). If multiple values a * requireMonthAndDay: `true` * Input values: - - Source: `[2003-12]` - - Target: `[2003-03-01]` + * Source: `[2003-12]` + * Target: `[2003-03-01]` * Returns: `Infinity` @@ -134,9 +138,9 @@ Compares single values (as opposed to sequences of values). If multiple values a If true, no distance value will be generated if months or days are missing (e.g., 2019-11). If false, missing month or day fields will default to 1. -- ID: `requireMonthAndDay` -- Datatype: `boolean` -- Default Value: `false` +* ID: `requireMonthAndDay` +* Datatype: `boolean` +* Default Value: `false` @@ -144,4 +148,4 @@ If true, no distance value will be generated if months or days are missing (e.g. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/dateTime.md b/docs/build/reference/distancemeasure/dateTime.md index 4807fe78..cbad0678 100644 --- a/docs/build/reference/distancemeasure/dateTime.md +++ b/docs/build/reference/distancemeasure/dateTime.md @@ -2,10 +2,12 @@ title: "DateTime" description: "Distance between two date time values (xsd:dateTime format) in seconds." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # DateTime + @@ -13,6 +15,7 @@ tags: Distance between two date time values (xsd:dateTime format) in seconds. ## Characteristics + This distance measure is not normalized, i.e., all distances start at 0 (exact match) and increase the more different the values are. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. @@ -23,4 +26,4 @@ Compares single values (as opposed to sequences of values). If multiple values a ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/dice.md b/docs/build/reference/distancemeasure/dice.md index b75d2436..aa6e2b36 100644 --- a/docs/build/reference/distancemeasure/dice.md +++ b/docs/build/reference/distancemeasure/dice.md @@ -2,10 +2,12 @@ title: "Dice coefficient" description: "Dice similarity coefficient." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Dice coefficient + @@ -13,6 +15,7 @@ tags: Dice similarity coefficient. ## Characteristics + This distance measure is normalized, i.e., all distances are between 0 (exact match) and 1 (no similarity). Compares sets of multiple values.Typically, incoming values are tokenized before being fed into this measure. @@ -23,4 +26,4 @@ Compares sets of multiple values.Typically, incoming values are tokenized before ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/equality.md b/docs/build/reference/distancemeasure/equality.md index 0d277b75..23db065f 100644 --- a/docs/build/reference/distancemeasure/equality.md +++ b/docs/build/reference/distancemeasure/equality.md @@ -2,10 +2,12 @@ title: "String equality" description: "Checks for equality of the string representation of the given values. Returns success if string values are equal, failure otherwise. For a numeric comparison of values use the 'Numeric Equality' comparator." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # String equality + @@ -13,9 +15,11 @@ tags: Checks for equality of the string representation of the given values. Returns success if string values are equal, failure otherwise. For a numeric comparison of values use the 'Numeric Equality' comparator. ## Characteristics + This is a boolean distance measure, i.e., all distances are either 0 or 1. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. + ## Examples **Notation:** List of values are represented via square brackets. Example: `[first, second]` represents a list of two values "first" and "second". @@ -23,21 +27,21 @@ Compares single values (as opposed to sequences of values). If multiple values a --- **Returns distance 0, if at least one value matches:** -* Input values: +- Input values: - Source: `[max, helmut]` - Target: `[max]` -* Returns: `0.0` +- Returns: `0.0` --- **Returns distance 1, if no value matches:** -* Input values: +- Input values: - Source: `[max, helmut]` - Target: `[john]` -* Returns: `1.0` +- Returns: `1.0` @@ -48,4 +52,4 @@ Compares single values (as opposed to sequences of values). If multiple values a ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/greaterThan.md b/docs/build/reference/distancemeasure/greaterThan.md index ea0d525d..dd1b009d 100644 --- a/docs/build/reference/distancemeasure/greaterThan.md +++ b/docs/build/reference/distancemeasure/greaterThan.md @@ -2,10 +2,12 @@ title: "Greater than" description: "Checks if the source value is greater than the target value. If both strings are numbers, numerical order is used for comparison. Otherwise, alphanumerical order is used." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Greater than + @@ -13,6 +15,7 @@ tags: Checks if the source value is greater than the target value. If both strings are numbers, numerical order is used for comparison. Otherwise, alphanumerical order is used. ## Characteristics + This is a boolean distance measure, i.e., all distances are either 0 or 1. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. @@ -52,4 +55,3 @@ Reverse source and target inputs - Default Value: `false` - diff --git a/docs/build/reference/distancemeasure/index.md b/docs/build/reference/distancemeasure/index.md index 4ddf1d11..95ddb0e4 100644 --- a/docs/build/reference/distancemeasure/index.md +++ b/docs/build/reference/distancemeasure/index.md @@ -5,7 +5,9 @@ tags: - Build - Reference --- + # Distance Measures + Distance Measures compute a distance metric between two sets of strings. diff --git a/docs/build/reference/distancemeasure/inequality.md b/docs/build/reference/distancemeasure/inequality.md index 1b58a101..20d26ac8 100644 --- a/docs/build/reference/distancemeasure/inequality.md +++ b/docs/build/reference/distancemeasure/inequality.md @@ -2,10 +2,12 @@ title: "Inequality" description: "Returns success if values are not equal, failure otherwise." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Inequality + @@ -13,9 +15,11 @@ tags: Returns success if values are not equal, failure otherwise. ## Characteristics + This is a boolean distance measure, i.e., all distances are either 0 or 1. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. + ## Examples **Notation:** List of values are represented via square brackets. Example: `[first, second]` represents a list of two values "first" and "second". @@ -23,41 +27,41 @@ Compares single values (as opposed to sequences of values). If multiple values a --- **Returns distance 0, if the values are different:** -* Input values: +- Input values: - Source: `[max]` - Target: `[john]` -* Returns: `0.0` +- Returns: `0.0` --- **Returns distance 1, if the values are equal:** -* Input values: +- Input values: - Source: `[max]` - Target: `[max]` -* Returns: `1.0` +- Returns: `1.0` --- **If multiple values are provided, returns 0, if at least one value does not match:** -* Input values: +- Input values: - Source: `[max, helmut]` - Target: `[max]` -* Returns: `0.0` +- Returns: `0.0` --- **If multiple values are provided, returns 1, if all value match:** -* Input values: +- Input values: - Source: `[max, max]` - Target: `[max, max]` -* Returns: `1.0` +- Returns: `1.0` @@ -68,4 +72,4 @@ Compares single values (as opposed to sequences of values). If multiple values a ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/insideNumericInterval.md b/docs/build/reference/distancemeasure/insideNumericInterval.md index 01eb235d..7cfa4f98 100644 --- a/docs/build/reference/distancemeasure/insideNumericInterval.md +++ b/docs/build/reference/distancemeasure/insideNumericInterval.md @@ -2,10 +2,12 @@ title: "Inside numeric interval" description: "Checks if a number is contained inside a numeric interval, such as '1900 - 2000'." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Inside numeric interval + @@ -13,6 +15,7 @@ tags: Checks if a number is contained inside a numeric interval, such as '1900 - 2000'. ## Characteristics + This is a boolean distance measure, i.e., all distances are either 0 or 1. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. @@ -33,4 +36,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/isSubstring.md b/docs/build/reference/distancemeasure/isSubstring.md index 9527bfb1..ca24e73c 100644 --- a/docs/build/reference/distancemeasure/isSubstring.md +++ b/docs/build/reference/distancemeasure/isSubstring.md @@ -2,10 +2,12 @@ title: "Is substring" description: "Checks if a source value is a substring of a target value." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Is substring + @@ -13,6 +15,7 @@ tags: Checks if a source value is a substring of a target value. ## Characteristics + This is a boolean distance measure, i.e., all distances are either 0 or 1. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. @@ -33,4 +36,4 @@ Reverse source and target inputs ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/jaccard.md b/docs/build/reference/distancemeasure/jaccard.md index 53d67e22..411f18e7 100644 --- a/docs/build/reference/distancemeasure/jaccard.md +++ b/docs/build/reference/distancemeasure/jaccard.md @@ -2,10 +2,12 @@ title: "Jaccard" description: "Jaccard similarity coefficient. Divides the matching tokens by the number of distinct tokens from both inputs." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Jaccard + @@ -13,9 +15,11 @@ tags: Jaccard similarity coefficient. Divides the matching tokens by the number of distinct tokens from both inputs. ## Characteristics + This distance measure is normalized, i.e., all distances are between 0 (exact match) and 1 (no similarity). Compares sets of multiple values.Typically, incoming values are tokenized before being fed into this measure. + ## Examples **Notation:** List of values are represented via square brackets. Example: `[first, second]` represents a list of two values "first" and "second". @@ -23,41 +27,41 @@ Compares sets of multiple values.Typically, incoming values are tokenized before --- **Returns 0 for equal sets of values:** -* Input values: +- Input values: - Source: `[A, B, C]` - Target: `[B, C, A]` -* Returns: `0.0` +- Returns: `0.0` --- **Returns 1 if there is no overlap between both sets of tokens:** -* Input values: +- Input values: - Source: `[A, B, C]` - Target: `[D, E, F]` -* Returns: `1.0` +- Returns: `1.0` --- **Returns 0.5 if half of all unique tokens overlap:** -* Input values: +- Input values: - Source: `[A, B, C]` - Target: `[A, B, D]` -* Returns: `0.5` +- Returns: `0.5` --- **Returns 2/3 if one third of all unique tokens overlap:** -* Input values: +- Input values: - Source: `[John, Jane]` - Target: `[John, Max]` -* Returns: `0.6666666666666666` +- Returns: `0.6666666666666666` @@ -68,4 +72,4 @@ Compares sets of multiple values.Typically, incoming values are tokenized before ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/jaro.md b/docs/build/reference/distancemeasure/jaro.md index d8e5cabe..7379bc34 100644 --- a/docs/build/reference/distancemeasure/jaro.md +++ b/docs/build/reference/distancemeasure/jaro.md @@ -2,19 +2,22 @@ title: "Jaro distance" description: "Matches strings based on the Jaro distance metric." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Jaro distance + -The Jaro distance measure calculates the similarity between two strings based on the number and order of common characters, the number of transpositions, and the length of the strings. The Jaro distance is 0 for a perfect match and 1 if there is no similarity between the given strings. +The Jaro distance measure calculates the similarity between two strings based on the number and order of common characters, the number of transpositions, and the length of the strings. The Jaro distance is 0 for a perfect match and 1 if there is no similarity between the given strings. For more information, please refer to: [https://en.wikipedia.org/wiki/Jaro–Winkler_distance](https://en.wikipedia.org/wiki/Jaro%E2%80%93Winkler_distance). ## Characteristics + This distance measure is normalized, i.e., all distances are between 0 (exact match) and 1 (no similarity). Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. @@ -25,4 +28,4 @@ Compares single values (as opposed to sequences of values). If multiple values a ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/jaroWinkler.md b/docs/build/reference/distancemeasure/jaroWinkler.md index e56b45a3..dd0bc65a 100644 --- a/docs/build/reference/distancemeasure/jaroWinkler.md +++ b/docs/build/reference/distancemeasure/jaroWinkler.md @@ -2,10 +2,12 @@ title: "Jaro-Winkler distance" description: "Matches strings based on the Jaro-Winkler distance measure." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Jaro-Winkler distance + @@ -15,6 +17,7 @@ The Jaro-Winkler distance measure is a variation of the Jaro distance metric. It For more information, please refer to: [https://en.wikipedia.org/wiki/Jaro–Winkler_distance](https://en.wikipedia.org/wiki/Jaro%E2%80%93Winkler_distance). ## Characteristics + This distance measure is normalized, i.e., all distances are between 0 (exact match) and 1 (no similarity). Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. @@ -25,4 +28,4 @@ Compares single values (as opposed to sequences of values). If multiple values a ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/koreanPhonemeDistance.md b/docs/build/reference/distancemeasure/koreanPhonemeDistance.md index dce4c5bd..05d64c3c 100644 --- a/docs/build/reference/distancemeasure/koreanPhonemeDistance.md +++ b/docs/build/reference/distancemeasure/koreanPhonemeDistance.md @@ -2,10 +2,12 @@ title: "Korean phoneme distance" description: "Korean phoneme distance." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Korean phoneme distance + @@ -13,6 +15,7 @@ tags: Korean phoneme distance. ## Characteristics + This distance measure is not normalized, i.e., all distances start at 0 (exact match) and increase the more different the values are. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. @@ -43,4 +46,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/koreanTranslitDistance.md b/docs/build/reference/distancemeasure/koreanTranslitDistance.md index 1000cf12..47447038 100644 --- a/docs/build/reference/distancemeasure/koreanTranslitDistance.md +++ b/docs/build/reference/distancemeasure/koreanTranslitDistance.md @@ -2,10 +2,12 @@ title: "Korean translit distance" description: "Transliterated Korean distance." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Korean translit distance + @@ -13,6 +15,7 @@ tags: Transliterated Korean distance. ## Characteristics + This distance measure is not normalized, i.e., all distances start at 0 (exact match) and increase the more different the values are. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. @@ -43,4 +46,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/levenshtein.md b/docs/build/reference/distancemeasure/levenshtein.md index f58a89d4..e1192727 100644 --- a/docs/build/reference/distancemeasure/levenshtein.md +++ b/docs/build/reference/distancemeasure/levenshtein.md @@ -2,10 +2,12 @@ title: "Normalized Levenshtein distance" description: "Normalized Levenshtein distance. Divides the edit distance by the length of the longer string." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Normalized Levenshtein distance + @@ -13,9 +15,11 @@ tags: Normalized Levenshtein distance. Divides the edit distance by the length of the longer string. ## Characteristics + This distance measure is normalized, i.e., all distances are between 0 (exact match) and 1 (no similarity). Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. + ## Examples **Notation:** List of values are represented via square brackets. Example: `[first, second]` represents a list of two values "first" and "second". @@ -23,41 +27,41 @@ Compares single values (as opposed to sequences of values). If multiple values a --- **Returns 0 for equal strings:** -* Input values: +- Input values: - Source: `[John]` - Target: `[John]` -* Returns: `0.0` +- Returns: `0.0` --- **Returns 1/4 if two strings of length 4 differ by one edit operation:** -* Input values: +- Input values: - Source: `[John]` - Target: `[Jxhn]` -* Returns: `0.25` +- Returns: `0.25` --- **Normalizes the edit distance by the length of the longer string:** -* Input values: +- Input values: - Source: `[John]` - Target: `[Jhn]` -* Returns: `0.25` +- Returns: `0.25` --- **Returns the maximum distance of 1 for completely different strings:** -* Input values: +- Input values: - Source: `[John]` - Target: `[Clara]` -* Returns: `1.0` +- Returns: `1.0` @@ -97,4 +101,3 @@ The maximum character that is used for indexing - Default Value: `z` - diff --git a/docs/build/reference/distancemeasure/levenshteinDistance.md b/docs/build/reference/distancemeasure/levenshteinDistance.md index 51dfe24e..688c5bda 100644 --- a/docs/build/reference/distancemeasure/levenshteinDistance.md +++ b/docs/build/reference/distancemeasure/levenshteinDistance.md @@ -2,10 +2,12 @@ title: "Levenshtein distance" description: "Levenshtein distance. Returns a distance value between zero and the size of the string." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Levenshtein distance + @@ -13,9 +15,11 @@ tags: Levenshtein distance. Returns a distance value between zero and the size of the string. ## Characteristics + This distance measure is not normalized, i.e., all distances start at 0 (exact match) and increase the more different the values are. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. + ## Examples **Notation:** List of values are represented via square brackets. Example: `[first, second]` represents a list of two values "first" and "second". @@ -23,31 +27,31 @@ Compares single values (as opposed to sequences of values). If multiple values a --- **Returns 0 for equal strings:** -* Input values: +- Input values: - Source: `[John]` - Target: `[John]` -* Returns: `0.0` +- Returns: `0.0` --- **Returns 1 for strings that differ by one edit operation:** -* Input values: +- Input values: - Source: `[John]` - Target: `[Jxhn]` -* Returns: `1.0` +- Returns: `1.0` --- **Returns 3 for strings that differ by three edit operations:** -* Input values: +- Input values: - Source: `[Saturday]` - Target: `[Sunday]` -* Returns: `3.0` +- Returns: `3.0` @@ -87,4 +91,3 @@ The maximum character that is used for indexing - Default Value: `z` - diff --git a/docs/build/reference/distancemeasure/lowerThan.md b/docs/build/reference/distancemeasure/lowerThan.md index f4e00577..39110e3f 100644 --- a/docs/build/reference/distancemeasure/lowerThan.md +++ b/docs/build/reference/distancemeasure/lowerThan.md @@ -2,10 +2,12 @@ title: "Lower than" description: "Checks if the source value is lower than the target value." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Lower than + @@ -13,6 +15,7 @@ tags: Checks if the source value is lower than the target value. ## Characteristics + This is a boolean distance measure, i.e., all distances are either 0 or 1. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. @@ -52,4 +55,3 @@ Reverse source and target inputs - Default Value: `false` - diff --git a/docs/build/reference/distancemeasure/num.md b/docs/build/reference/distancemeasure/num.md index 8e480203..1a0249bb 100644 --- a/docs/build/reference/distancemeasure/num.md +++ b/docs/build/reference/distancemeasure/num.md @@ -2,10 +2,12 @@ title: "Numeric similarity" description: "Computes the numeric distance between two numbers." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Numeric similarity + @@ -13,6 +15,7 @@ tags: Computes the numeric distance between two numbers. ## Characteristics + This distance measure is not normalized, i.e., all distances start at 0 (exact match) and increase the more different the values are. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. @@ -42,4 +45,3 @@ The maximum number that is used for indexing - Default Value: `Infinity` - diff --git a/docs/build/reference/distancemeasure/numericEquality.md b/docs/build/reference/distancemeasure/numericEquality.md index ae134739..e0efb23f 100644 --- a/docs/build/reference/distancemeasure/numericEquality.md +++ b/docs/build/reference/distancemeasure/numericEquality.md @@ -2,10 +2,12 @@ title: "Numeric equality" description: "Compares values numerically instead of their string representation as the 'String Equality' operator does. Allows to set the needed precision of the comparison. A value of 0.0 means that the values must represent exactly the same (floating point) value, values higher than that allow for a margin of tolerance." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Numeric equality + @@ -15,9 +17,11 @@ Allows to set the needed precision of the comparison. A value of 0.0 means that (floating point) value, values higher than that allow for a margin of tolerance. ## Characteristics + This is a boolean distance measure, i.e., all distances are either 0 or 1. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. + ## Examples **Notation:** List of values are represented via square brackets. Example: `[first, second]` represents a list of two values "first" and "second". @@ -26,8 +30,8 @@ Compares single values (as opposed to sequences of values). If multiple values a **Returns 0 for equal numbers:** * Input values: - - Source: `[4.2]` - - Target: `[4.2]` + * Source: `[4.2]` + * Target: `[4.2]` * Returns: `0.0` @@ -36,8 +40,8 @@ Compares single values (as opposed to sequences of values). If multiple values a **Returns 1 if at least one value is not a number:** * Input values: - - Source: `[1]` - - Target: `[one]` + * Source: `[1]` + * Target: `[one]` * Returns: `1.0` @@ -49,8 +53,8 @@ Compares single values (as opposed to sequences of values). If multiple values a * precision: `0.1` * Input values: - - Source: `[1.3]` - - Target: `[1.35]` + * Source: `[1.3]` + * Target: `[1.35]` * Returns: `0.0` @@ -62,8 +66,8 @@ Compares single values (as opposed to sequences of values). If multiple values a * precision: `0.1` * Input values: - - Source: `[1.3]` - - Target: `[1.5]` + * Source: `[1.3]` + * Target: `[1.5]` * Returns: `1.0` @@ -76,9 +80,9 @@ Compares single values (as opposed to sequences of values). If multiple values a The range of tolerance in floating point number comparisons. Must be 0 or a non-negative number smaller than 1. -- ID: `precision` -- Datatype: `double` -- Default Value: `0.0` +* ID: `precision` +* Datatype: `double` +* Default Value: `0.0` @@ -86,4 +90,4 @@ The range of tolerance in floating point number comparisons. Must be 0 or a non- ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/qGrams.md b/docs/build/reference/distancemeasure/qGrams.md index 2ee49cad..34181d26 100644 --- a/docs/build/reference/distancemeasure/qGrams.md +++ b/docs/build/reference/distancemeasure/qGrams.md @@ -2,10 +2,12 @@ title: "qGrams" description: "String similarity based on q-grams (by default q=2)." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # qGrams + @@ -13,9 +15,11 @@ tags: String similarity based on q-grams (by default q=2). ## Characteristics + This distance measure is normalized, i.e., all distances are between 0 (exact match) and 1 (no similarity). Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. + ## Examples **Notation:** List of values are represented via square brackets. Example: `[first, second]` represents a list of two values "first" and "second". @@ -23,31 +27,31 @@ Compares single values (as opposed to sequences of values). If multiple values a --- **Returns 0.0 if the input strings are equal:** -* Input values: +- Input values: - Source: `[abcd]` - Target: `[abcd]` -* Returns: `0.0` +- Returns: `0.0` --- **Returns 1.0 if the input strings do not share a single q-gram:** -* Input values: +- Input values: - Source: `[abcd]` - Target: `[dcba]` -* Returns: `1.0` +- Returns: `1.0` --- **Returns 1 minus the matching q-grams divided by the total number of q-grams. Generated q-grams in this example: (#a, ab, b#) and (#a, ac, c#):** -* Input values: +- Input values: - Source: `[ab]` - Target: `[ac]` -* Returns: `0.8` +- Returns: `0.8` @@ -87,4 +91,3 @@ The maximum character that is used for indexing - Default Value: `z` - diff --git a/docs/build/reference/distancemeasure/relaxedEquality.md b/docs/build/reference/distancemeasure/relaxedEquality.md index 6c826fcf..c819a6fb 100644 --- a/docs/build/reference/distancemeasure/relaxedEquality.md +++ b/docs/build/reference/distancemeasure/relaxedEquality.md @@ -2,10 +2,12 @@ title: "Relaxed equality" description: "Return success if strings are equal, failure otherwise. Lower/upper case and differences like ö/o, n/ñ, c/ç etc. are treated as equal." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Relaxed equality + @@ -13,6 +15,7 @@ tags: Return success if strings are equal, failure otherwise. Lower/upper case and differences like ö/o, n/ñ, c/ç etc. are treated as equal. ## Characteristics + This is a boolean distance measure, i.e., all distances are either 0 or 1. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. @@ -23,4 +26,4 @@ Compares single values (as opposed to sequences of values). If multiple values a ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/softjaccard.md b/docs/build/reference/distancemeasure/softjaccard.md index 9bff4ecc..e96f5993 100644 --- a/docs/build/reference/distancemeasure/softjaccard.md +++ b/docs/build/reference/distancemeasure/softjaccard.md @@ -2,10 +2,12 @@ title: "Soft Jaccard" description: "Soft Jaccard similarity coefficient. Same as Jaccard distance but values within an levenhstein distance of 'maxDistance' are considered equivalent." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Soft Jaccard + @@ -13,6 +15,7 @@ tags: Soft Jaccard similarity coefficient. Same as Jaccard distance but values within an levenhstein distance of 'maxDistance' are considered equivalent. ## Characteristics + This distance measure is normalized, i.e., all distances are between 0 (exact match) and 1 (no similarity). Compares sets of multiple values.Typically, incoming values are tokenized before being fed into this measure. @@ -33,4 +36,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/startsWith.md b/docs/build/reference/distancemeasure/startsWith.md index 4edc7fe7..720170ca 100644 --- a/docs/build/reference/distancemeasure/startsWith.md +++ b/docs/build/reference/distancemeasure/startsWith.md @@ -2,10 +2,12 @@ title: "Starts with" description: "Returns success if the first string starts with the second string, failure otherwise." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Starts with + @@ -13,6 +15,7 @@ tags: Returns success if the first string starts with the second string, failure otherwise. ## Characteristics + This is a boolean distance measure, i.e., all distances are either 0 or 1. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. @@ -53,4 +56,4 @@ The potential maximum length of the strings that must match. If the max length i ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/substringDistance.md b/docs/build/reference/distancemeasure/substringDistance.md index c7ba8095..1a8b0704 100644 --- a/docs/build/reference/distancemeasure/substringDistance.md +++ b/docs/build/reference/distancemeasure/substringDistance.md @@ -2,10 +2,12 @@ title: "Substring comparison" description: "Return 0 to 1 for strong similarity to weak similarity. Based on the paper: Stoilos, Giorgos, Giorgos Stamou, and Stefanos Kollias. 'A string metric for ontology alignment.' The Semantic Web-ISWC 2005. Springer Berlin Heidelberg, 2005. 624-637." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Substring comparison + @@ -13,6 +15,7 @@ tags: Return 0 to 1 for strong similarity to weak similarity. Based on the paper: Stoilos, Giorgos, Giorgos Stamou, and Stefanos Kollias. "A string metric for ontology alignment." The Semantic Web-ISWC 2005. Springer Berlin Heidelberg, 2005. 624-637. ## Characteristics + This distance measure is normalized, i.e., all distances are between 0 (exact match) and 1 (no similarity). Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. @@ -33,4 +36,4 @@ The minimum length of a possible substring match. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/distancemeasure/tokenwiseDistance.md b/docs/build/reference/distancemeasure/tokenwiseDistance.md index e8111ef8..07997280 100644 --- a/docs/build/reference/distancemeasure/tokenwiseDistance.md +++ b/docs/build/reference/distancemeasure/tokenwiseDistance.md @@ -2,10 +2,12 @@ title: "Token-wise distance" description: "Token-wise string distance using the specified metric." icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Token-wise distance + @@ -13,6 +15,7 @@ tags: Token-wise string distance using the specified metric. ## Characteristics + This distance measure is normalized, i.e., all distances are between 0 (exact match) and 1 (no similarity). Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. @@ -122,4 +125,3 @@ Use incremental IDF weights - Default Value: `false` - diff --git a/docs/build/reference/distancemeasure/wgs84.md b/docs/build/reference/distancemeasure/wgs84.md index a57fd857..93817456 100644 --- a/docs/build/reference/distancemeasure/wgs84.md +++ b/docs/build/reference/distancemeasure/wgs84.md @@ -2,10 +2,12 @@ title: "Geographical distance" description: "Computes the geographical distance between two points. Author: Konrad Höffner (MOLE subgroup of Research Group AKSW, University of Leipzig)" icon: octicons/cross-reference-24 -tags: +tags: - DistanceMeasure --- + # Geographical distance + @@ -13,6 +15,7 @@ tags: Computes the geographical distance between two points. Author: Konrad Höffner (MOLE subgroup of Research Group AKSW, University of Leipzig) ## Characteristics + This distance measure is not normalized, i.e., all distances start at 0 (exact match) and increase the more different the values are. Compares single values (as opposed to sequences of values). If multiple values are provided, all values are compared and the lowest distance is returned. @@ -33,4 +36,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/index.md b/docs/build/reference/index.md index 3aa61a3e..98ecf5b8 100644 --- a/docs/build/reference/index.md +++ b/docs/build/reference/index.md @@ -5,41 +5,43 @@ tags: - Build - Reference --- + # Task and Operator Reference + **:octicons-people-24: Intended audience:** Linked Data Experts and Domain Experts
-- [Aggregators](aggregator/index.md) +- [Aggregators](aggregator/index.md) --- This kind of task aggregates multiple similarity scores. -- [Custom Workflow Tasks](customtask/index.md) +- [Custom Workflow Tasks](customtask/index.md) --- An operator that can be used in a workflow. -- [Datasets](dataset/index.md) +- [Datasets](dataset/index.md) --- A collection of data that can be read or written. -- [Distance Measures](distancemeasure/index.md) +- [Distance Measures](distancemeasure/index.md) --- Computes the distance between two sets of strings. -- [Transformer](transformer/index.md) +- [Transformer](transformer/index.md) --- Transforms a sequence of string values. -
\ No newline at end of file + diff --git a/docs/build/reference/transformer/Combine/concat.md b/docs/build/reference/transformer/Combine/concat.md index d90e9892..884fcf76 100644 --- a/docs/build/reference/transformer/Combine/concat.md +++ b/docs/build/reference/transformer/Combine/concat.md @@ -2,10 +2,12 @@ title: "Concatenate" description: "Concatenates strings from multiple inputs." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Concatenate + @@ -120,8 +122,8 @@ Concatenates strings from multiple inputs. 1. `[First]` 2. `[Second]` -* Returns: - ``` +* Returns: + ```text [First Second] ``` @@ -137,7 +139,7 @@ Concatenates strings from multiple inputs. 1. `[First]` 2. `[Second]` -* Returns: `[First \\aSecond]` +* Returns: `[First \\aSecond]` @@ -148,9 +150,9 @@ Concatenates strings from multiple inputs. Separator to be inserted between two concatenated strings. The text can contain escaped characters \n, \t and \\ that are replaced by a newline, tab or backslash respectively. -- ID: `glue` -- Datatype: `string` -- Default Value: `None` +* ID: `glue` +* Datatype: `string` +* Default Value: `None` @@ -158,9 +160,9 @@ Separator to be inserted between two concatenated strings. The text can contain Handle missing values as empty strings. -- ID: `missingValuesAsEmptyStrings` -- Datatype: `boolean` -- Default Value: `false` +* ID: `missingValuesAsEmptyStrings` +* Datatype: `boolean` +* Default Value: `false` @@ -168,4 +170,4 @@ Handle missing values as empty strings. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Combine/concatMultiValues.md b/docs/build/reference/transformer/Combine/concatMultiValues.md index ce3e33c6..c1ee5141 100644 --- a/docs/build/reference/transformer/Combine/concatMultiValues.md +++ b/docs/build/reference/transformer/Combine/concatMultiValues.md @@ -2,10 +2,12 @@ title: "Concatenate multiple values" description: "Concatenates multiple values received for an input. If applied to multiple inputs, yields at most one value per input. Optionally removes duplicate values." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Concatenate multiple values + @@ -69,14 +71,14 @@ Concatenates multiple values received for an input. If applied to multiple input * glue: `\n\t\\` * Input values: - 1. - ``` + 1. + ```text [a \b, c] ``` -* Returns: - ``` +* Returns: + ```text [a \b \c] @@ -91,9 +93,9 @@ Concatenates multiple values received for an input. If applied to multiple input No description -- ID: `glue` -- Datatype: `string` -- Default Value: `None` +* ID: `glue` +* Datatype: `string` +* Default Value: `None` @@ -101,9 +103,9 @@ No description No description -- ID: `removeDuplicates` -- Datatype: `boolean` -- Default Value: `false` +* ID: `removeDuplicates` +* Datatype: `boolean` +* Default Value: `false` @@ -111,4 +113,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Combine/concatPairwise.md b/docs/build/reference/transformer/Combine/concatPairwise.md index 3a073b5f..776efc4e 100644 --- a/docs/build/reference/transformer/Combine/concatPairwise.md +++ b/docs/build/reference/transformer/Combine/concatPairwise.md @@ -2,10 +2,12 @@ title: "Concatenate pairwise" description: "Concatenates the values of multiple inputs pairwise." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Concatenate pairwise + @@ -70,9 +72,9 @@ Concatenates the values of multiple inputs pairwise. Separator to be inserted between two concatenated strings. The text can contain escaped characters \n, \t and \\ that are replaced by a newline, tab or backslash respectively. -- ID: `glue` -- Datatype: `string` -- Default Value: `None` +* ID: `glue` +* Datatype: `string` +* Default Value: `None` @@ -80,4 +82,4 @@ Separator to be inserted between two concatenated strings. The text can contain ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Combine/merge.md b/docs/build/reference/transformer/Combine/merge.md index afaec47d..4fda6c7c 100644 --- a/docs/build/reference/transformer/Combine/merge.md +++ b/docs/build/reference/transformer/Combine/merge.md @@ -2,10 +2,12 @@ title: "Merge" description: "Merges the values of all inputs." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Merge + @@ -40,4 +42,4 @@ Merges the values of all inputs. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Combine/zip.md b/docs/build/reference/transformer/Combine/zip.md index 27282da8..68a0202e 100644 --- a/docs/build/reference/transformer/Combine/zip.md +++ b/docs/build/reference/transformer/Combine/zip.md @@ -2,10 +2,12 @@ title: "Zip" description: "Concatenates the values of two inputs in pairs." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Zip + @@ -63,8 +65,8 @@ Concatenates the values of two inputs in pairs. 1. `[a, b]` 2. `[1, 2]` -* Returns: - ``` +* Returns: + ```text [a 1, b 2] @@ -79,9 +81,9 @@ Concatenates the values of two inputs in pairs. Placeholder to be used if the first input provides fewer values than the second one. -- ID: `firstPlaceholder` -- Datatype: `string` -- Default Value: `None` +* ID: `firstPlaceholder` +* Datatype: `string` +* Default Value: `None` @@ -89,9 +91,9 @@ Placeholder to be used if the first input provides fewer values than the second Placeholder to be used if the second input provides fewer values than the first one. -- ID: `secondPlaceholder` -- Datatype: `string` -- Default Value: `None` +* ID: `secondPlaceholder` +* Datatype: `string` +* Default Value: `None` @@ -99,9 +101,9 @@ Placeholder to be used if the second input provides fewer values than the first Separator to be inserted between two concatenated strings. The text can contain escaped characters \n, \t and \\ that are replaced by a newline, tab or backslash respectively. -- ID: `glue` -- Datatype: `string` -- Default Value: `None` +* ID: `glue` +* Datatype: `string` +* Default Value: `None` @@ -109,4 +111,4 @@ Separator to be inserted between two concatenated strings. The text can contain ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Conditional/containsAllOf.md b/docs/build/reference/transformer/Conditional/containsAllOf.md index e5811f74..6a5f1af0 100644 --- a/docs/build/reference/transformer/Conditional/containsAllOf.md +++ b/docs/build/reference/transformer/Conditional/containsAllOf.md @@ -2,10 +2,12 @@ title: "Contains all of" description: "Accepts two inputs. If the first input contains all of the second input values it returns 'true', else 'false' is returned." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Contains all of + @@ -97,4 +99,4 @@ Accepts two inputs. If the first input contains all of the second input values i ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Conditional/containsAnyOf.md b/docs/build/reference/transformer/Conditional/containsAnyOf.md index a57df728..50c4030a 100644 --- a/docs/build/reference/transformer/Conditional/containsAnyOf.md +++ b/docs/build/reference/transformer/Conditional/containsAnyOf.md @@ -2,10 +2,12 @@ title: "Contains any of" description: "Accepts two inputs. If the first input contains any of the second input values it returns 'true', else 'false' is returned." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Contains any of + @@ -97,4 +99,4 @@ Accepts two inputs. If the first input contains any of the second input values i ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Conditional/ifContains.md b/docs/build/reference/transformer/Conditional/ifContains.md index c53abc8c..05c38136 100644 --- a/docs/build/reference/transformer/Conditional/ifContains.md +++ b/docs/build/reference/transformer/Conditional/ifContains.md @@ -2,10 +2,12 @@ title: "If contains" description: "Accepts two or three inputs. If the first input contains the given value, the second input is forwarded. Otherwise, the third input is forwarded (if present)." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # If contains + @@ -64,9 +66,9 @@ Accepts two or three inputs. If the first input contains the given value, the se No description -- ID: `search` -- Datatype: `string` -- Default Value: `None` +* ID: `search` +* Datatype: `string` +* Default Value: `None` @@ -74,4 +76,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Conditional/ifExists.md b/docs/build/reference/transformer/Conditional/ifExists.md index 8c554b52..f6a60a40 100644 --- a/docs/build/reference/transformer/Conditional/ifExists.md +++ b/docs/build/reference/transformer/Conditional/ifExists.md @@ -2,10 +2,12 @@ title: "If exists" description: "Accepts two or three inputs. If the first input provides a value, the second input is forwarded. Otherwise, the third input is forwarded (if present)." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # If exists + @@ -56,4 +58,4 @@ Accepts two or three inputs. If the first input provides a value, the second inp ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Conditional/ifMatchesRegex.md b/docs/build/reference/transformer/Conditional/ifMatchesRegex.md index 6310c287..1e17514f 100644 --- a/docs/build/reference/transformer/Conditional/ifMatchesRegex.md +++ b/docs/build/reference/transformer/Conditional/ifMatchesRegex.md @@ -2,10 +2,12 @@ title: "If matches regex" description: "This transformer uses a regular expression as a matching condition, in order to distinguish which input to take." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # If matches regex + @@ -114,9 +116,9 @@ A compilation of the available constructs for building regular expressions is av No description -- ID: `regex` -- Datatype: `string` -- Default Value: `None` +* ID: `regex` +* Datatype: `string` +* Default Value: `None` @@ -124,9 +126,9 @@ No description No description -- ID: `negate` -- Datatype: `boolean` -- Default Value: `false` +* ID: `negate` +* Datatype: `boolean` +* Default Value: `false` @@ -134,4 +136,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Conditional/negateTransformer.md b/docs/build/reference/transformer/Conditional/negateTransformer.md index aa357a59..59bb1772 100644 --- a/docs/build/reference/transformer/Conditional/negateTransformer.md +++ b/docs/build/reference/transformer/Conditional/negateTransformer.md @@ -2,10 +2,12 @@ title: "Negate binary (NOT)" description: "Accepts one input, which is either 'true', '1' or 'false', '0' and negates it." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Negate binary (NOT) + @@ -53,4 +55,4 @@ Accepts one input, which is either 'true', '1' or 'false', '0' and negates it. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Conversion/convertCharset.md b/docs/build/reference/transformer/Conversion/convertCharset.md index c4aa0d2d..675c1a06 100644 --- a/docs/build/reference/transformer/Conversion/convertCharset.md +++ b/docs/build/reference/transformer/Conversion/convertCharset.md @@ -2,10 +2,12 @@ title: "Convert charset" description: "Convert the string from 'sourceCharset' to 'targetCharset'." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Convert charset + @@ -39,4 +41,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Date/compareDates.md b/docs/build/reference/transformer/Date/compareDates.md index b343fd53..198937d5 100644 --- a/docs/build/reference/transformer/Date/compareDates.md +++ b/docs/build/reference/transformer/Date/compareDates.md @@ -2,10 +2,12 @@ title: "Compare dates" description: "Compares two dates." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Compare dates + @@ -139,9 +141,9 @@ So invalid values do not make the plugin fail. They are simply ignored, and if n No description -- ID: `comparator` -- Datatype: `enumeration` -- Default Value: `<` +* ID: `comparator` +* Datatype: `enumeration` +* Default Value: `<` @@ -149,4 +151,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Date/currentDate.md b/docs/build/reference/transformer/Date/currentDate.md index f199ae24..bb7df60c 100644 --- a/docs/build/reference/transformer/Date/currentDate.md +++ b/docs/build/reference/transformer/Date/currentDate.md @@ -2,10 +2,12 @@ title: "Current date" description: "Outputs the current date." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Current date + @@ -19,4 +21,4 @@ Outputs the current date. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Date/datetoTimestamp.md b/docs/build/reference/transformer/Date/datetoTimestamp.md index 498ffbd3..e6a8f265 100644 --- a/docs/build/reference/transformer/Date/datetoTimestamp.md +++ b/docs/build/reference/transformer/Date/datetoTimestamp.md @@ -2,10 +2,12 @@ title: "Date to timestamp" description: "Convert an xsd:dateTime to a timestamp. Returns the passed time since the Unix Epoch (1970-01-01)." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Date to timestamp + @@ -63,9 +65,9 @@ Convert an xsd:dateTime to a timestamp. Returns the passed time since the Unix E No description -- ID: `unit` -- Datatype: `enumeration` -- Default Value: `milliseconds` +* ID: `unit` +* Datatype: `enumeration` +* Default Value: `milliseconds` @@ -73,4 +75,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Date/duration.md b/docs/build/reference/transformer/Date/duration.md index 804d37f2..ba292f49 100644 --- a/docs/build/reference/transformer/Date/duration.md +++ b/docs/build/reference/transformer/Date/duration.md @@ -2,10 +2,12 @@ title: "Duration" description: "Computes the time difference between two data times." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Duration + @@ -19,4 +21,4 @@ Computes the time difference between two data times. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Date/durationInDays.md b/docs/build/reference/transformer/Date/durationInDays.md index 9ca73deb..56fb5cb7 100644 --- a/docs/build/reference/transformer/Date/durationInDays.md +++ b/docs/build/reference/transformer/Date/durationInDays.md @@ -2,10 +2,12 @@ title: "Duration in days" description: "Converts an xsd:duration to days." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Duration in days + @@ -19,4 +21,4 @@ Converts an xsd:duration to days. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Date/durationInSeconds.md b/docs/build/reference/transformer/Date/durationInSeconds.md index 846ee351..367dad59 100644 --- a/docs/build/reference/transformer/Date/durationInSeconds.md +++ b/docs/build/reference/transformer/Date/durationInSeconds.md @@ -2,10 +2,12 @@ title: "Duration in seconds" description: "Converts an xsd:duration to seconds." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Duration in seconds + @@ -19,4 +21,4 @@ Converts an xsd:duration to seconds. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Date/durationInYears.md b/docs/build/reference/transformer/Date/durationInYears.md index 3bf9a5e5..ffc09f8d 100644 --- a/docs/build/reference/transformer/Date/durationInYears.md +++ b/docs/build/reference/transformer/Date/durationInYears.md @@ -2,10 +2,12 @@ title: "Duration in years" description: "Converts an xsd:duration to years." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Duration in years + @@ -19,4 +21,4 @@ Converts an xsd:duration to years. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Date/numberToDuration.md b/docs/build/reference/transformer/Date/numberToDuration.md index bcea640a..e3ec6283 100644 --- a/docs/build/reference/transformer/Date/numberToDuration.md +++ b/docs/build/reference/transformer/Date/numberToDuration.md @@ -2,10 +2,12 @@ title: "Number to duration" description: "Converts a number to an xsd:duration." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Number to duration + @@ -29,4 +31,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Date/parseDate.md b/docs/build/reference/transformer/Date/parseDate.md index 15c7dc92..c38a81ba 100644 --- a/docs/build/reference/transformer/Date/parseDate.md +++ b/docs/build/reference/transformer/Date/parseDate.md @@ -2,10 +2,12 @@ title: "Parse date pattern" description: "Parses a date based on a specified pattern, returning an xsd:date." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Parse date pattern + @@ -114,9 +116,9 @@ Parses a date based on a specified pattern, returning an xsd:date. The date pattern used to parse the input values -- ID: `format` -- Datatype: `string` -- Default Value: `dd-MM-yyyy` +* ID: `format` +* Datatype: `string` +* Default Value: `dd-MM-yyyy` @@ -124,9 +126,9 @@ The date pattern used to parse the input values If set to true, the parser tries to use heuristics to parse dates with invalid fields (such as a day of zero). -- ID: `lenient` -- Datatype: `boolean` -- Default Value: `false` +* ID: `lenient` +* Datatype: `boolean` +* Default Value: `false` @@ -134,9 +136,9 @@ If set to true, the parser tries to use heuristics to parse dates with invalid f Optional locale for the date format. If not set the system's locale will be used. -- ID: `locale` -- Datatype: `option[locale]` -- Default Value: `None` +* ID: `locale` +* Datatype: `option[locale]` +* Default Value: `None` @@ -144,4 +146,4 @@ Optional locale for the date format. If not set the system's locale will be used ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Date/timeToDate.md b/docs/build/reference/transformer/Date/timeToDate.md index a06ae064..de6d09fc 100644 --- a/docs/build/reference/transformer/Date/timeToDate.md +++ b/docs/build/reference/transformer/Date/timeToDate.md @@ -2,10 +2,12 @@ title: "Timestamp to date" description: "Convert a timestamp to xsd:date format. Expects an integer that denotes the passed time since the Unix Epoch (1970-01-01)" icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Timestamp to date + @@ -58,9 +60,9 @@ Convert a timestamp to xsd:date format. Expects an integer that denotes the pass Custom output format (e.g., 'yyyy-MM-dd'). If left empty, a full xsd:dateTime (UTC) is returned. -- ID: `format` -- Datatype: `string` -- Default Value: `None` +* ID: `format` +* Datatype: `string` +* Default Value: `None` @@ -68,9 +70,9 @@ Custom output format (e.g., 'yyyy-MM-dd'). If left empty, a full xsd:dateTime (U No description -- ID: `unit` -- Datatype: `enumeration` -- Default Value: `milliseconds` +* ID: `unit` +* Datatype: `enumeration` +* Default Value: `milliseconds` @@ -78,4 +80,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_ABS.md b/docs/build/reference/transformer/Excel/Excel_ABS.md index e551f485..0fe40a82 100644 --- a/docs/build/reference/transformer/Excel/Excel_ABS.md +++ b/docs/build/reference/transformer/Excel/Excel_ABS.md @@ -2,10 +2,12 @@ title: "Abs" description: "Excel ABS(number): Returns the absolute value of the given number." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Abs + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_ACOS.md b/docs/build/reference/transformer/Excel/Excel_ACOS.md index a7cc6a0b..ebfaa99c 100644 --- a/docs/build/reference/transformer/Excel/Excel_ACOS.md +++ b/docs/build/reference/transformer/Excel/Excel_ACOS.md @@ -2,10 +2,12 @@ title: "Acos" description: "Excel ACOS(number): Returns the inverse cosine of the given number in radians." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Acos + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_ACOSH.md b/docs/build/reference/transformer/Excel/Excel_ACOSH.md index e6f6a941..ee0ff597 100644 --- a/docs/build/reference/transformer/Excel/Excel_ACOSH.md +++ b/docs/build/reference/transformer/Excel/Excel_ACOSH.md @@ -2,10 +2,12 @@ title: "Acosh" description: "Excel ACOSH(number): Returns the inverse hyperbolic cosine of the given number in radians." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Acosh + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_AND.md b/docs/build/reference/transformer/Excel/Excel_AND.md index 869f644d..2c0cceb7 100644 --- a/docs/build/reference/transformer/Excel/Excel_AND.md +++ b/docs/build/reference/transformer/Excel/Excel_AND.md @@ -2,10 +2,12 @@ title: "And" description: "Excel AND(argument1; argument2 ...argument30): Returns TRUE if all the arguments are considered TRUE, and FALSE otherwise." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # And + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_ASIN.md b/docs/build/reference/transformer/Excel/Excel_ASIN.md index d0b2cd3f..eb7eff76 100644 --- a/docs/build/reference/transformer/Excel/Excel_ASIN.md +++ b/docs/build/reference/transformer/Excel/Excel_ASIN.md @@ -2,10 +2,12 @@ title: "Asin" description: "Excel ASIN(number): Returns the inverse sine of the given number in radians." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Asin + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_ASINH.md b/docs/build/reference/transformer/Excel/Excel_ASINH.md index 5aa2653b..8fe6cde6 100644 --- a/docs/build/reference/transformer/Excel/Excel_ASINH.md +++ b/docs/build/reference/transformer/Excel/Excel_ASINH.md @@ -2,10 +2,12 @@ title: "Asinh" description: "Excel ASINH(number): Returns the inverse hyperbolic sine of the given number in radians." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Asinh + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_ATAN.md b/docs/build/reference/transformer/Excel/Excel_ATAN.md index b4852b82..4e1b2e57 100644 --- a/docs/build/reference/transformer/Excel/Excel_ATAN.md +++ b/docs/build/reference/transformer/Excel/Excel_ATAN.md @@ -2,10 +2,12 @@ title: "Atan" description: "Excel ATAN(number): Returns the inverse tangent of the given number in radians." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Atan + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_ATAN2.md b/docs/build/reference/transformer/Excel/Excel_ATAN2.md index 81215550..ebcb8acb 100644 --- a/docs/build/reference/transformer/Excel/Excel_ATAN2.md +++ b/docs/build/reference/transformer/Excel/Excel_ATAN2.md @@ -2,10 +2,12 @@ title: "Atan2" description: "Excel ATAN2(number_x; number_y): Returns the inverse tangent of the specified x and y coordinates. Number_x is the value for the x coordinate. Number_y is the value for the y coordinate." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Atan2 + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_ATANH.md b/docs/build/reference/transformer/Excel/Excel_ATANH.md index 0f807ace..4839db7d 100644 --- a/docs/build/reference/transformer/Excel/Excel_ATANH.md +++ b/docs/build/reference/transformer/Excel/Excel_ATANH.md @@ -2,10 +2,12 @@ title: "Atanh" description: "Excel ATANH(number): Returns the inverse hyperbolic tangent of the given number. (Angle is returned in radians.)" icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Atanh + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_AVEDEV.md b/docs/build/reference/transformer/Excel/Excel_AVEDEV.md index 27ba796b..d22c0ef6 100644 --- a/docs/build/reference/transformer/Excel/Excel_AVEDEV.md +++ b/docs/build/reference/transformer/Excel/Excel_AVEDEV.md @@ -2,10 +2,12 @@ title: "Avedev" description: "Excel AVEDEV(number1; number2; ... number_30): Returns the average of the absolute deviations of data points from their mean. Displays the diffusion in a data set. Number_1; number_2; ... number_30 are values or ranges that represent a sample. Each number can also be replaced by a reference." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Avedev + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_AVERAGE.md b/docs/build/reference/transformer/Excel/Excel_AVERAGE.md index 680bb380..040b01fa 100644 --- a/docs/build/reference/transformer/Excel/Excel_AVERAGE.md +++ b/docs/build/reference/transformer/Excel/Excel_AVERAGE.md @@ -2,10 +2,12 @@ title: "Average" description: "Excel AVERAGE(number_1; number_2; ... number_30): Returns the average of the arguments. Number_1; number_2; ... number_30 are numerical values or ranges. Text is ignored." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Average + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_AVERAGEA.md b/docs/build/reference/transformer/Excel/Excel_AVERAGEA.md index d38574e5..13e288cf 100644 --- a/docs/build/reference/transformer/Excel/Excel_AVERAGEA.md +++ b/docs/build/reference/transformer/Excel/Excel_AVERAGEA.md @@ -2,10 +2,12 @@ title: "Averagea" description: "Excel AVERAGEA(value_1; value_2; ... value_30): Returns the average of the arguments. The value of a text is 0. Value_1; value_2; ... value_30 are values or ranges." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Averagea + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_CEILING.md b/docs/build/reference/transformer/Excel/Excel_CEILING.md index 7b6717a1..c518eda5 100644 --- a/docs/build/reference/transformer/Excel/Excel_CEILING.md +++ b/docs/build/reference/transformer/Excel/Excel_CEILING.md @@ -2,10 +2,12 @@ title: "Ceiling" description: "Excel CEILING(number; significance; mode): Rounds the given number to the nearest integer or multiple of significance. Significance is the value to whose multiple of ten the value is to be rounded up (.01, .1, 1, 10, etc.). Mode is an optional value. If it is indicated and non-zero and if the number and significance are negative, rounding up is carried out based on that value." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Ceiling + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_CHOOSE.md b/docs/build/reference/transformer/Excel/Excel_CHOOSE.md index 41bdddf7..102252a1 100644 --- a/docs/build/reference/transformer/Excel/Excel_CHOOSE.md +++ b/docs/build/reference/transformer/Excel/Excel_CHOOSE.md @@ -2,10 +2,12 @@ title: "Choose" description: "Excel CHOOSE(index; value1; ... value30): Uses an index to return a value from a list of up to 30 values. Index is a reference or number between 1 and 30 indicating which value is to be taken from the list. Value1; ... value30 is the list of values entered as a reference to a cell or as individual values." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Choose + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_CLEAN.md b/docs/build/reference/transformer/Excel/Excel_CLEAN.md index f94dd7c4..1117aa26 100644 --- a/docs/build/reference/transformer/Excel/Excel_CLEAN.md +++ b/docs/build/reference/transformer/Excel/Excel_CLEAN.md @@ -2,10 +2,12 @@ title: "Clean" description: "Excel CLEAN(text): Removes all non-printing characters from the string. Text refers to the text from which to remove all non-printable characters." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Clean + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_CODE.md b/docs/build/reference/transformer/Excel/Excel_CODE.md index a01209e9..9c4d439c 100644 --- a/docs/build/reference/transformer/Excel/Excel_CODE.md +++ b/docs/build/reference/transformer/Excel/Excel_CODE.md @@ -2,10 +2,12 @@ title: "Code" description: "Excel CODE(text): Returns a numeric code for the first character in a text string. Text is the text for which the code of the first character is to be found." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Code + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_COMBIN.md b/docs/build/reference/transformer/Excel/Excel_COMBIN.md index 1ee4b618..b805885c 100644 --- a/docs/build/reference/transformer/Excel/Excel_COMBIN.md +++ b/docs/build/reference/transformer/Excel/Excel_COMBIN.md @@ -2,10 +2,12 @@ title: "Combin" description: "Excel COMBIN(count_1; count_2): Returns the number of combinations for a given number of objects. Count_1 is the total number of elements. Count_2 is the selected count from the elements. This is the same as the nCr function on a calculator." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Combin + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_CORREL.md b/docs/build/reference/transformer/Excel/Excel_CORREL.md index 9c23fd23..ce345fc7 100644 --- a/docs/build/reference/transformer/Excel/Excel_CORREL.md +++ b/docs/build/reference/transformer/Excel/Excel_CORREL.md @@ -2,10 +2,12 @@ title: "Correl" description: "Excel CORREL(data_1; data_2): Returns the correlation coefficient between two data sets. Data_1 is the first data set. Data_2 is the second data set." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Correl + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_COS.md b/docs/build/reference/transformer/Excel/Excel_COS.md index 48fed071..86fedf62 100644 --- a/docs/build/reference/transformer/Excel/Excel_COS.md +++ b/docs/build/reference/transformer/Excel/Excel_COS.md @@ -2,10 +2,12 @@ title: "Cos" description: "Excel COS(number): Returns the cosine of the given number (angle in radians)." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Cos + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_COSH.md b/docs/build/reference/transformer/Excel/Excel_COSH.md index 7b71113c..d0c1c6de 100644 --- a/docs/build/reference/transformer/Excel/Excel_COSH.md +++ b/docs/build/reference/transformer/Excel/Excel_COSH.md @@ -2,10 +2,12 @@ title: "Cosh" description: "Excel COSH(number): Returns the hyperbolic cosine of the given number (angle in radians)." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Cosh + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_COUNT.md b/docs/build/reference/transformer/Excel/Excel_COUNT.md index 43f7ad29..8a2f70ae 100644 --- a/docs/build/reference/transformer/Excel/Excel_COUNT.md +++ b/docs/build/reference/transformer/Excel/Excel_COUNT.md @@ -2,10 +2,12 @@ title: "Count" description: "Excel COUNT(value_1; value_2; ... value_30): Counts how many numbers are in the list of arguments. Text entries are ignored. Value_1; value_2; ... value_30 are values or ranges which are to be counted." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Count + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_COUNTA.md b/docs/build/reference/transformer/Excel/Excel_COUNTA.md index 599ecf98..fcc8bb3b 100644 --- a/docs/build/reference/transformer/Excel/Excel_COUNTA.md +++ b/docs/build/reference/transformer/Excel/Excel_COUNTA.md @@ -2,10 +2,12 @@ title: "Counta" description: "Excel COUNTA(value_1; value_2; ... value_30): Counts how many values are in the list of arguments. Text entries are also counted, even when they contain an empty string of length 0. If an argument is an array or reference, empty cells within the array or reference are ignored. value_1; value_2; ... value_30 are up to 30 arguments representing the values to be counted." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Counta + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_COVAR.md b/docs/build/reference/transformer/Excel/Excel_COVAR.md index 7246beb2..7a28df7c 100644 --- a/docs/build/reference/transformer/Excel/Excel_COVAR.md +++ b/docs/build/reference/transformer/Excel/Excel_COVAR.md @@ -2,10 +2,12 @@ title: "Covar" description: "Excel COVAR(data_1; data_2): Returns the covariance of the product of paired deviations. Data_1 is the first data set. Data_2 is the second data set." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Covar + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_DEGREES.md b/docs/build/reference/transformer/Excel/Excel_DEGREES.md index 6a991ece..b78eb87e 100644 --- a/docs/build/reference/transformer/Excel/Excel_DEGREES.md +++ b/docs/build/reference/transformer/Excel/Excel_DEGREES.md @@ -2,10 +2,12 @@ title: "Degrees" description: "Excel DEGREES(number): Converts the given number in radians to degrees." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Degrees + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_DEVSQ.md b/docs/build/reference/transformer/Excel/Excel_DEVSQ.md index 7e5cf6e5..f0a4b007 100644 --- a/docs/build/reference/transformer/Excel/Excel_DEVSQ.md +++ b/docs/build/reference/transformer/Excel/Excel_DEVSQ.md @@ -2,10 +2,12 @@ title: "Devsq" description: "Excel DEVSQ(number_1; number_2; ... number_30): Returns the sum of squares of deviations based on a sample mean. Number_1; number_2; ... number_30 are numerical values or ranges representing a sample." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Devsq + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_EVEN.md b/docs/build/reference/transformer/Excel/Excel_EVEN.md index 788e36c6..620d0a1d 100644 --- a/docs/build/reference/transformer/Excel/Excel_EVEN.md +++ b/docs/build/reference/transformer/Excel/Excel_EVEN.md @@ -2,10 +2,12 @@ title: "Even" description: "Excel EVEN(number): Rounds the given number up to the nearest even integer." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Even + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_EXACT.md b/docs/build/reference/transformer/Excel/Excel_EXACT.md index fc6e8aaa..92599f40 100644 --- a/docs/build/reference/transformer/Excel/Excel_EXACT.md +++ b/docs/build/reference/transformer/Excel/Excel_EXACT.md @@ -2,10 +2,12 @@ title: "Exact" description: "Excel EXACT(text_1; text_2): Compares two text strings and returns TRUE if they are identical. This function is case- sensitive. Text_1 is the first text to compare. Text_2 is the second text to compare." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Exact + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_EXP.md b/docs/build/reference/transformer/Excel/Excel_EXP.md index 98898854..d7378bdc 100644 --- a/docs/build/reference/transformer/Excel/Excel_EXP.md +++ b/docs/build/reference/transformer/Excel/Excel_EXP.md @@ -2,10 +2,12 @@ title: "Exp" description: "Excel EXP(number): Returns e raised to the power of the given number." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Exp + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_FACT.md b/docs/build/reference/transformer/Excel/Excel_FACT.md index 6bcf0b85..5214f377 100644 --- a/docs/build/reference/transformer/Excel/Excel_FACT.md +++ b/docs/build/reference/transformer/Excel/Excel_FACT.md @@ -2,10 +2,12 @@ title: "Fact" description: "Excel FACT(number): Returns the factorial of the given number." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Fact + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_FALSE.md b/docs/build/reference/transformer/Excel/Excel_FALSE.md index 8df870cf..f643ff22 100644 --- a/docs/build/reference/transformer/Excel/Excel_FALSE.md +++ b/docs/build/reference/transformer/Excel/Excel_FALSE.md @@ -2,10 +2,12 @@ title: "False" description: "Excel FALSE(): Set the logical value to FALSE. The FALSE() function does not require any arguments." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # False + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_FIND.md b/docs/build/reference/transformer/Excel/Excel_FIND.md index 97cbb663..b1f05da4 100644 --- a/docs/build/reference/transformer/Excel/Excel_FIND.md +++ b/docs/build/reference/transformer/Excel/Excel_FIND.md @@ -2,10 +2,12 @@ title: "Find" description: "Excel FIND(find_text; text; position): Looks for a string of text within another string. Where to begin the search can also be defined. The search term can be a number or any string of characters. The search is case-sensitive. Find_text is the text to be found. Text is the text where the search takes place. Position (optional) is the position in the text from which the search starts." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Find + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_FLOOR.md b/docs/build/reference/transformer/Excel/Excel_FLOOR.md index 1940904f..c5f81a74 100644 --- a/docs/build/reference/transformer/Excel/Excel_FLOOR.md +++ b/docs/build/reference/transformer/Excel/Excel_FLOOR.md @@ -2,10 +2,12 @@ title: "Floor" description: "Excel FLOOR(number; significance; mode): Rounds the given number down to the nearest multiple of significance. Significance is the value to whose multiple of ten the number is to be rounded down (.01, .1, 1, 10, etc.). Mode is an optional value. If it is indicated and non-zero and if the number and significance are negative, rounding up is carried out based on that value." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Floor + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_FORECAST.md b/docs/build/reference/transformer/Excel/Excel_FORECAST.md index 518d0a84..72d1553f 100644 --- a/docs/build/reference/transformer/Excel/Excel_FORECAST.md +++ b/docs/build/reference/transformer/Excel/Excel_FORECAST.md @@ -2,10 +2,12 @@ title: "Forecast" description: "Excel FORECAST(value; data_Y; data_X): Extrapolates future values based on existing x and y values. Value is the x value, for which the y value of the linear regression is to be returned. Data_Y is the array or range of known y's. Data_X is the array or range of known x's. Does not work for exponential functions." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Forecast + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_FV.md b/docs/build/reference/transformer/Excel/Excel_FV.md index 1be99e2f..46ad43a6 100644 --- a/docs/build/reference/transformer/Excel/Excel_FV.md +++ b/docs/build/reference/transformer/Excel/Excel_FV.md @@ -2,10 +2,12 @@ title: "Fv" description: "Excel FV(rate; NPER; PMT; PV; type): Returns the future value of an investment based on periodic, constant payments and a constant interest rate. Rate is the periodic interest rate. NPER is the total number of periods. PMT is the annuity paid regularly per period. PV (optional) is the present cash value of an investment. Type (optional) defines whether the payment is due at the beginning (1) or the end (0) of a period." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Fv + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_GEOMEAN.md b/docs/build/reference/transformer/Excel/Excel_GEOMEAN.md index 72d1cb6f..9180aefd 100644 --- a/docs/build/reference/transformer/Excel/Excel_GEOMEAN.md +++ b/docs/build/reference/transformer/Excel/Excel_GEOMEAN.md @@ -2,10 +2,12 @@ title: "Geomean" description: "Excel GEOMEAN(number_1; number_2; ... number_30): Returns the geometric mean of a sample. Number_1; number_2; ... number_30 are numerical arguments or ranges that represent a random sample." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Geomean + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_IF.md b/docs/build/reference/transformer/Excel/Excel_IF.md index 2654c3d6..f05d1504 100644 --- a/docs/build/reference/transformer/Excel/Excel_IF.md +++ b/docs/build/reference/transformer/Excel/Excel_IF.md @@ -2,10 +2,12 @@ title: "If" description: "Excel IF(test; then_value; otherwise_value): Returns different values based on the test value. Note that in this implementation it will not actually evaluate logical conditions. Then_value is the value that is returned if the test is TRUE. Otherwise_value (optional) is the value that is returned if the test is FALSE." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # If + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_INT.md b/docs/build/reference/transformer/Excel/Excel_INT.md index 5ad7f870..99139d21 100644 --- a/docs/build/reference/transformer/Excel/Excel_INT.md +++ b/docs/build/reference/transformer/Excel/Excel_INT.md @@ -2,10 +2,12 @@ title: "Int" description: "Excel INT(number): Rounds the given number down to the nearest integer." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Int + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_INTERCEPT.md b/docs/build/reference/transformer/Excel/Excel_INTERCEPT.md index fa4d859f..51c09f5c 100644 --- a/docs/build/reference/transformer/Excel/Excel_INTERCEPT.md +++ b/docs/build/reference/transformer/Excel/Excel_INTERCEPT.md @@ -2,10 +2,12 @@ title: "Intercept" description: "Excel INTERCEPT(data_Y; data_X): Calculates the y-value at which a line will intersect the y-axis by using known x-values and y-values. Data_Y is the dependent set of observations or data. Data_X is the independent set of observations or data. Names, arrays or references containing numbers must be used here. Numbers can also be entered directly." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Intercept + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_IPMT.md b/docs/build/reference/transformer/Excel/Excel_IPMT.md index a09184aa..9bb100c3 100644 --- a/docs/build/reference/transformer/Excel/Excel_IPMT.md +++ b/docs/build/reference/transformer/Excel/Excel_IPMT.md @@ -2,10 +2,12 @@ title: "Ipmt" description: "Excel IPMT(rate; period; NPER; PV; FV; type): Calculates the periodic amortization for an investment with regular payments and a constant interest rate. Rate is the periodic interest rate. Period is the period for which the compound interest is calculated. NPER is the total number of periods during which annuity is paid. Period=NPER, if compound interest for the last period is calculated. PV is the present cash value in sequence of payments. FV (optional) is the desired value (future value) at the end of the periods. Type (optional) defines whether the payment is due at the beginning (1) or the end (0) of a period." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Ipmt + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_IRR.md b/docs/build/reference/transformer/Excel/Excel_IRR.md index 090e35c3..67954d6f 100644 --- a/docs/build/reference/transformer/Excel/Excel_IRR.md +++ b/docs/build/reference/transformer/Excel/Excel_IRR.md @@ -2,10 +2,12 @@ title: "Irr" description: "Excel IRR(values; guess): Calculates the internal rate of return for an investment. The values represent cash flow values at regular intervals; at least one value must be negative (payments), and at least one value must be positive (income). Values is an array containing the values. Guess (optional) is the estimated value. If you can provide only a few values, you should provide an initial guess to enable the iteration." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Irr + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_LARGE.md b/docs/build/reference/transformer/Excel/Excel_LARGE.md index 3f3f3c4a..35f7a290 100644 --- a/docs/build/reference/transformer/Excel/Excel_LARGE.md +++ b/docs/build/reference/transformer/Excel/Excel_LARGE.md @@ -2,10 +2,12 @@ title: "Large" description: "Excel LARGE(data; rank_c): Returns the Rank_c-th largest value in a data set. Data is the cell range of data. Rank_c is the ranking of the value (2nd largest, 3rd largest, etc.) written as an integer." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Large + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_LEFT.md b/docs/build/reference/transformer/Excel/Excel_LEFT.md index d32e50ff..9cbe8a2c 100644 --- a/docs/build/reference/transformer/Excel/Excel_LEFT.md +++ b/docs/build/reference/transformer/Excel/Excel_LEFT.md @@ -2,10 +2,12 @@ title: "Left" description: "Excel LEFT(text; number): Returns the first character or characters in a text string. Text is the text where the initial partial words are to be determined. Number (optional) is the number of characters for the start text. If this parameter is not defined, one character is returned." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Left + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_LN.md b/docs/build/reference/transformer/Excel/Excel_LN.md index 69f6959c..1f468498 100644 --- a/docs/build/reference/transformer/Excel/Excel_LN.md +++ b/docs/build/reference/transformer/Excel/Excel_LN.md @@ -2,10 +2,12 @@ title: "Ln" description: "Excel LN(number): Returns the natural logarithm based on the constant e of the given number." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Ln + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_LOG.md b/docs/build/reference/transformer/Excel/Excel_LOG.md index 5c4c37b8..142d64a1 100644 --- a/docs/build/reference/transformer/Excel/Excel_LOG.md +++ b/docs/build/reference/transformer/Excel/Excel_LOG.md @@ -2,10 +2,12 @@ title: "Log" description: "Excel LOG(number; base): Returns the logarithm of the given number to the specified base. Base is the base for the logarithm calculation." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Log + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_LOG10.md b/docs/build/reference/transformer/Excel/Excel_LOG10.md index 844c0cbe..4017151e 100644 --- a/docs/build/reference/transformer/Excel/Excel_LOG10.md +++ b/docs/build/reference/transformer/Excel/Excel_LOG10.md @@ -2,10 +2,12 @@ title: "Log10" description: "Excel LOG10(number): Returns the base-10 logarithm of the given number." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Log10 + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_MAX.md b/docs/build/reference/transformer/Excel/Excel_MAX.md index db8f0852..816155ee 100644 --- a/docs/build/reference/transformer/Excel/Excel_MAX.md +++ b/docs/build/reference/transformer/Excel/Excel_MAX.md @@ -2,10 +2,12 @@ title: "Max" description: "Excel MAX(number_1; number_2; ... number_30): Returns the maximum value in a list of arguments. Number_1; number_2; ... number_30 are numerical values or ranges." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Max + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_MAXA.md b/docs/build/reference/transformer/Excel/Excel_MAXA.md index 6c18e0af..d2f9c686 100644 --- a/docs/build/reference/transformer/Excel/Excel_MAXA.md +++ b/docs/build/reference/transformer/Excel/Excel_MAXA.md @@ -2,10 +2,12 @@ title: "Maxa" description: "Excel MAXA(value_1; value_2; ... value_30): Returns the maximum value in a list of arguments. Unlike MAX, text can be entered. The value of the text is 0. Value_1; value_2; ... value_30 are values or ranges." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Maxa + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_MEDIAN.md b/docs/build/reference/transformer/Excel/Excel_MEDIAN.md index 78c96327..e5994e1f 100644 --- a/docs/build/reference/transformer/Excel/Excel_MEDIAN.md +++ b/docs/build/reference/transformer/Excel/Excel_MEDIAN.md @@ -2,10 +2,12 @@ title: "Median" description: "Excel MEDIAN(number_1; number_2; ... number_30): Returns the median of a set of numbers. Number_1; number_2; ... number_30 are values or ranges, which represent a sample. Each number can also be replaced by a reference." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Median + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_MID.md b/docs/build/reference/transformer/Excel/Excel_MID.md index 9f8d8764..3c3895b7 100644 --- a/docs/build/reference/transformer/Excel/Excel_MID.md +++ b/docs/build/reference/transformer/Excel/Excel_MID.md @@ -2,10 +2,12 @@ title: "Mid" description: "Excel MID(text; start; number): Returns a text segment of a character string. The parameters specify the starting position and the number of characters. Text is the text containing the characters to extract. Start is the position of the first character in the text to extract. Number is the number of characters in the part of the text." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Mid + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_MIN.md b/docs/build/reference/transformer/Excel/Excel_MIN.md index 5193a07b..b92c91ea 100644 --- a/docs/build/reference/transformer/Excel/Excel_MIN.md +++ b/docs/build/reference/transformer/Excel/Excel_MIN.md @@ -2,10 +2,12 @@ title: "Min" description: "Excel MIN(number_1; number_2; ... number_30): Returns the minimum value in a list of arguments. Number_1; number_2; ... number_30 are numerical values or ranges." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Min + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_MINA.md b/docs/build/reference/transformer/Excel/Excel_MINA.md index 8c8c07bb..88646b31 100644 --- a/docs/build/reference/transformer/Excel/Excel_MINA.md +++ b/docs/build/reference/transformer/Excel/Excel_MINA.md @@ -2,10 +2,12 @@ title: "Mina" description: "Excel MINA(value_1; value_2; ... value_30): Returns the minimum value in a list of arguments. Here text can also be entered. The value of the text is 0. Value_1; value_2; ... value_30 are values or ranges." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Mina + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_MIRR.md b/docs/build/reference/transformer/Excel/Excel_MIRR.md index 39adb915..ad9c3d7c 100644 --- a/docs/build/reference/transformer/Excel/Excel_MIRR.md +++ b/docs/build/reference/transformer/Excel/Excel_MIRR.md @@ -2,10 +2,12 @@ title: "Mirr" description: "Excel MIRR(values; investment; reinvest_rate): Calculates the modified internal rate of return of a series of investments. Values corresponds to the array or the cell reference for cells whose content corresponds to the payments. Investment is the rate of interest of the investments (the negative values of the array) Reinvest_rate is the rate of interest of the reinvestment (the positive values of the array)." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Mirr + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_MOD.md b/docs/build/reference/transformer/Excel/Excel_MOD.md index 3c6fe204..c5c2c5cf 100644 --- a/docs/build/reference/transformer/Excel/Excel_MOD.md +++ b/docs/build/reference/transformer/Excel/Excel_MOD.md @@ -2,10 +2,12 @@ title: "Mod" description: "Excel MOD(dividend; divisor): Returns the remainder after a number is divided by a divisor. Dividend is the number which will be divided by the divisor. Divisor is the number by which to divide the dividend." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Mod + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_MODE.md b/docs/build/reference/transformer/Excel/Excel_MODE.md index 3363d2dc..2b835258 100644 --- a/docs/build/reference/transformer/Excel/Excel_MODE.md +++ b/docs/build/reference/transformer/Excel/Excel_MODE.md @@ -2,10 +2,12 @@ title: "Mode" description: "Excel MODE(number_1; number_2; ... number_30): Returns the most common value in a data set. Number_1; number_2; ... number_30 are numerical values or ranges. If several values have the same frequency, it returns the smallest value. An error occurs when a value does not appear twice." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Mode + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_NORMDIST.md b/docs/build/reference/transformer/Excel/Excel_NORMDIST.md index 2fcbf79e..429767e7 100644 --- a/docs/build/reference/transformer/Excel/Excel_NORMDIST.md +++ b/docs/build/reference/transformer/Excel/Excel_NORMDIST.md @@ -2,10 +2,12 @@ title: "Normdist" description: "Excel NORMDIST(number; mean; STDEV; C): Returns the normal distribution for the given Number in the distribution. Mean is the mean value of the distribution. STDEV is the standard deviation of the distribution. C = 0 calculates the density function, and C = 1 calculates the distribution." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Normdist + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_NORMINV.md b/docs/build/reference/transformer/Excel/Excel_NORMINV.md index d8cdb283..9b666ef6 100644 --- a/docs/build/reference/transformer/Excel/Excel_NORMINV.md +++ b/docs/build/reference/transformer/Excel/Excel_NORMINV.md @@ -2,10 +2,12 @@ title: "Norminv" description: "Excel NORMINV(number; mean; STDEV): Returns the inverse of the normal distribution for the given Number in the distribution. Mean is the mean value in the normal distribution. STDEV is the standard deviation of the normal distribution." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Norminv + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_NORMSDIST.md b/docs/build/reference/transformer/Excel/Excel_NORMSDIST.md index 4df1241c..b4f78709 100644 --- a/docs/build/reference/transformer/Excel/Excel_NORMSDIST.md +++ b/docs/build/reference/transformer/Excel/Excel_NORMSDIST.md @@ -2,10 +2,12 @@ title: "Normsdist" description: "Excel NORMSDIST(number): Returns the standard normal cumulative distribution for the given Number." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Normsdist + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_NORMSINV.md b/docs/build/reference/transformer/Excel/Excel_NORMSINV.md index d23dcca3..4c0069fe 100644 --- a/docs/build/reference/transformer/Excel/Excel_NORMSINV.md +++ b/docs/build/reference/transformer/Excel/Excel_NORMSINV.md @@ -2,10 +2,12 @@ title: "Normsinv" description: "Excel NORMSINV(number): Returns the inverse of the standard normal distribution for the given Number, a probability value." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Normsinv + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_NOT.md b/docs/build/reference/transformer/Excel/Excel_NOT.md index c9befb75..318b5035 100644 --- a/docs/build/reference/transformer/Excel/Excel_NOT.md +++ b/docs/build/reference/transformer/Excel/Excel_NOT.md @@ -2,10 +2,12 @@ title: "Not" description: "Excel NOT(logical_value): Reverses the logical value. Logical_value is any value to be reversed." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Not + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_NPER.md b/docs/build/reference/transformer/Excel/Excel_NPER.md index 0a44af65..4294ab36 100644 --- a/docs/build/reference/transformer/Excel/Excel_NPER.md +++ b/docs/build/reference/transformer/Excel/Excel_NPER.md @@ -2,10 +2,12 @@ title: "Nper" description: "Excel NPER(rate; PMT; PV; FV; type): Returns the number of periods for an investment based on periodic, constant payments and a constant interest rate. Rate is the periodic interest rate. PMT is the constant annuity paid in each period. PV is the present value (cash value) in a sequence of payments. FV (optional) is the future value, which is reached at the end of the last period. Type (optional) defines whether the payment is due at the beginning (1) or the end (0) of a period." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Nper + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_NPV.md b/docs/build/reference/transformer/Excel/Excel_NPV.md index 08609248..7c8c0d94 100644 --- a/docs/build/reference/transformer/Excel/Excel_NPV.md +++ b/docs/build/reference/transformer/Excel/Excel_NPV.md @@ -2,10 +2,12 @@ title: "Npv" description: "Excel NPV(Rate; value_1; value_2; ... value_30): Returns the net present value of an investment based on a series of periodic cash flows and a discount rate. Rate is the discount rate for a period. Value_1; value_2;... value_30 are values representing deposits or withdrawals." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Npv + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_ODD.md b/docs/build/reference/transformer/Excel/Excel_ODD.md index 9080f415..ba608cc7 100644 --- a/docs/build/reference/transformer/Excel/Excel_ODD.md +++ b/docs/build/reference/transformer/Excel/Excel_ODD.md @@ -2,10 +2,12 @@ title: "Odd" description: "Excel ODD(number): Rounds the given number up to the nearest odd integer." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Odd + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_OR.md b/docs/build/reference/transformer/Excel/Excel_OR.md index 12079437..69f6b24f 100644 --- a/docs/build/reference/transformer/Excel/Excel_OR.md +++ b/docs/build/reference/transformer/Excel/Excel_OR.md @@ -2,10 +2,12 @@ title: "Or" description: "Excel OR(logical_value_1; logical_value_2; ...logical_value_30): Returns TRUE if at least one argument is TRUE. Returns the value FALSE if all the arguments have the logical value FALSE. Logical_value_1; logical_value_2; ...logical_value_30 are conditions to be checked. All conditions can be either TRUE or FALSE. If a range is entered as a parameter, the function uses the value from the range that is in the current column or row." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Or + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_PEARSON.md b/docs/build/reference/transformer/Excel/Excel_PEARSON.md index 5ce586e6..3b13b416 100644 --- a/docs/build/reference/transformer/Excel/Excel_PEARSON.md +++ b/docs/build/reference/transformer/Excel/Excel_PEARSON.md @@ -2,10 +2,12 @@ title: "Pearson" description: "Excel PEARSON(data_1; data_2): Returns the Pearson product moment correlation coefficient r. Data_1 is the array of the first data set. Data_2 is the array of the second data set." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Pearson + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_PERCENTILE.md b/docs/build/reference/transformer/Excel/Excel_PERCENTILE.md index 279c6bc2..9f946bfc 100644 --- a/docs/build/reference/transformer/Excel/Excel_PERCENTILE.md +++ b/docs/build/reference/transformer/Excel/Excel_PERCENTILE.md @@ -2,10 +2,12 @@ title: "Percentile" description: "Excel PERCENTILE(data; alpha): Returns the alpha-percentile of data values in an array. Data is the array of data. Alpha is the percentage of the scale between 0 and 1." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Percentile + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_PERCENTRANK.md b/docs/build/reference/transformer/Excel/Excel_PERCENTRANK.md index ebf7be19..33eca799 100644 --- a/docs/build/reference/transformer/Excel/Excel_PERCENTRANK.md +++ b/docs/build/reference/transformer/Excel/Excel_PERCENTRANK.md @@ -2,10 +2,12 @@ title: "Percentrank" description: "Excel PERCENTRANK(data; value): Returns the percentage rank (percentile) of the given value in a sample. Data is the array of data in the sample." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Percentrank + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_PI.md b/docs/build/reference/transformer/Excel/Excel_PI.md index 5e501067..565ad3c9 100644 --- a/docs/build/reference/transformer/Excel/Excel_PI.md +++ b/docs/build/reference/transformer/Excel/Excel_PI.md @@ -2,10 +2,12 @@ title: "Pi" description: "Excel PI(): Returns the value of PI to fourteen decimal places." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Pi + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_PMT.md b/docs/build/reference/transformer/Excel/Excel_PMT.md index 9fd91e49..60a65d07 100644 --- a/docs/build/reference/transformer/Excel/Excel_PMT.md +++ b/docs/build/reference/transformer/Excel/Excel_PMT.md @@ -2,10 +2,12 @@ title: "Pmt" description: "Excel PMT(rate; NPER; PV; FV; type): Returns the periodic payment for an annuity with constant interest rates. Rate is the periodic interest rate. NPER is the number of periods in which annuity is paid. PV is the present value (cash value) in a sequence of payments. FV (optional) is the desired value (future value) to be reached at the end of the periodic payments. Type (optional) defines whether the payment is due at the beginning (1) or the end (0) of a period." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Pmt + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_POISSON.md b/docs/build/reference/transformer/Excel/Excel_POISSON.md index 89695131..8fc45ccd 100644 --- a/docs/build/reference/transformer/Excel/Excel_POISSON.md +++ b/docs/build/reference/transformer/Excel/Excel_POISSON.md @@ -2,10 +2,12 @@ title: "Poisson" description: "Excel POISSON(number; mean; C): Returns the Poisson distribution for the given Number. Mean is the middle value of the Poisson distribution. C = 0 calculates the density function, and C = 1 calculates the distribution." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Poisson + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_POWER.md b/docs/build/reference/transformer/Excel/Excel_POWER.md index 2a8ab348..7aadfcc4 100644 --- a/docs/build/reference/transformer/Excel/Excel_POWER.md +++ b/docs/build/reference/transformer/Excel/Excel_POWER.md @@ -2,10 +2,12 @@ title: "Power" description: "Excel POWER(base; power): Returns the result of a number raised to a power. Base is the number that is to be raised to the given power. Power is the exponent by which the base is to be raised." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Power + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_PPMT.md b/docs/build/reference/transformer/Excel/Excel_PPMT.md index 9dcbf59a..59af7b71 100644 --- a/docs/build/reference/transformer/Excel/Excel_PPMT.md +++ b/docs/build/reference/transformer/Excel/Excel_PPMT.md @@ -2,10 +2,12 @@ title: "Ppmt" description: "Excel PPMT(rate; period; NPER; PV; FV; type): Returns for a given period the payment on the principal for an investment that is based on periodic and constant payments and a constant interest rate. Rate is the periodic interest rate. Period is the amortization period. NPER is the total number of periods during which annuity is paid. PV is the present value in the sequence of payments. FV (optional) is the desired (future) value. Type (optional) defines whether the payment is due at the beginning (1) or the end (0) of a period." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Ppmt + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_PRODUCT.md b/docs/build/reference/transformer/Excel/Excel_PRODUCT.md index a0a2a32a..a24aff55 100644 --- a/docs/build/reference/transformer/Excel/Excel_PRODUCT.md +++ b/docs/build/reference/transformer/Excel/Excel_PRODUCT.md @@ -2,10 +2,12 @@ title: "Product" description: "Excel PRODUCT(number 1 to 30): Multiplies all the numbers given as arguments and returns the product. Number 1 to number 30 are up to 30 arguments whose product is to be calculated, separated by semi-colons." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Product + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_PROPER.md b/docs/build/reference/transformer/Excel/Excel_PROPER.md index 473236ce..574de602 100644 --- a/docs/build/reference/transformer/Excel/Excel_PROPER.md +++ b/docs/build/reference/transformer/Excel/Excel_PROPER.md @@ -2,10 +2,12 @@ title: "Proper" description: "Excel PROPER(text): Capitalizes the first letter in all words of a text string. Text is the text to be converted." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Proper + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_PV.md b/docs/build/reference/transformer/Excel/Excel_PV.md index 54213526..b2135f58 100644 --- a/docs/build/reference/transformer/Excel/Excel_PV.md +++ b/docs/build/reference/transformer/Excel/Excel_PV.md @@ -2,10 +2,12 @@ title: "Pv" description: "Excel PV(rate; NPER; PMT; FV; type): Returns the present value of an investment resulting from a series of regular payments. Rate defines the interest rate per period. NPER is the total number of payment periods. PMT is the regular payment made per period. FV (optional) defines the future value remaining after the final installment has been made. Type (optional) defines whether the payment is due at the beginning (1) or the end (0) of a period." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Pv + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_RADIANS.md b/docs/build/reference/transformer/Excel/Excel_RADIANS.md index 1dea5c36..4fbec9ac 100644 --- a/docs/build/reference/transformer/Excel/Excel_RADIANS.md +++ b/docs/build/reference/transformer/Excel/Excel_RADIANS.md @@ -2,10 +2,12 @@ title: "Radians" description: "Excel RADIANS(number): Converts the given number in degrees to radians." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Radians + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_RAND.md b/docs/build/reference/transformer/Excel/Excel_RAND.md index 770a83a4..0e49ff6d 100644 --- a/docs/build/reference/transformer/Excel/Excel_RAND.md +++ b/docs/build/reference/transformer/Excel/Excel_RAND.md @@ -2,10 +2,12 @@ title: "Rand" description: "Excel RAND(): Returns a random number between 0 and 1." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Rand + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_RANK.md b/docs/build/reference/transformer/Excel/Excel_RANK.md index dfcbd839..17dadbf1 100644 --- a/docs/build/reference/transformer/Excel/Excel_RANK.md +++ b/docs/build/reference/transformer/Excel/Excel_RANK.md @@ -2,10 +2,12 @@ title: "Rank" description: "Excel RANK(value; data; type): Returns the rank of the given Value in a sample. Data is the array or range of data in the sample. Type (optional) is the sequence order, either ascending (0) or descending (1)." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Rank + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_RATE.md b/docs/build/reference/transformer/Excel/Excel_RATE.md index a6c3b994..53d2e847 100644 --- a/docs/build/reference/transformer/Excel/Excel_RATE.md +++ b/docs/build/reference/transformer/Excel/Excel_RATE.md @@ -2,10 +2,12 @@ title: "Rate" description: "Excel RATE(NPER; PMT; PV; FV; type; guess): Returns the constant interest rate per period of an annuity. NPER is the total number of periods, during which payments are made (payment period). PMT is the constant payment (annuity) paid during each period. PV is the cash value in the sequence of payments. FV (optional) is the future value, which is reached at the end of the periodic payments. Type (optional) defines whether the payment is due at the beginning (1) or the end (0) of a period. Guess (optional) determines the estimated value of the interest with iterative calculation." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Rate + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_REPLACE.md b/docs/build/reference/transformer/Excel/Excel_REPLACE.md index 1b78f81f..d42f92ce 100644 --- a/docs/build/reference/transformer/Excel/Excel_REPLACE.md +++ b/docs/build/reference/transformer/Excel/Excel_REPLACE.md @@ -2,10 +2,12 @@ title: "Replace" description: "Excel REPLACE(text; position; length; new_text): Replaces part of a text string with a different text string. This function can be used to replace both characters and numbers (which are automatically converted to text). The result of the function is always displayed as text. To perform further calculations with a number which has been replaced by text, convert it back to a number using the VALUE function. Any text containing numbers must be enclosed in quotation marks so it is not interpreted as a number and automatically converted to text. Text is text of which a part will be replaced. Position is the position within the text where the replacement will begin. Length is the number of characters in text to be replaced. New_text is the text which replaces text.." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Replace + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_REPT.md b/docs/build/reference/transformer/Excel/Excel_REPT.md index 360c81d6..77e441ca 100644 --- a/docs/build/reference/transformer/Excel/Excel_REPT.md +++ b/docs/build/reference/transformer/Excel/Excel_REPT.md @@ -2,10 +2,12 @@ title: "Rept" description: "Excel REPT(text; number): Repeats a character string by the given number of copies. Text is the text to be repeated. Number is the number of repetitions. The result can be a maximum of 255 characters." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Rept + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_RIGHT.md b/docs/build/reference/transformer/Excel/Excel_RIGHT.md index 8ad63b8e..85085b8f 100644 --- a/docs/build/reference/transformer/Excel/Excel_RIGHT.md +++ b/docs/build/reference/transformer/Excel/Excel_RIGHT.md @@ -2,10 +2,12 @@ title: "Right" description: "Excel RIGHT(text; number): Defines the last character or characters in a text string. Text is the text of which the right part is to be determined. Number (optional) is the number of characters from the right part of the text." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Right + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_ROMAN.md b/docs/build/reference/transformer/Excel/Excel_ROMAN.md index 03e4e6af..92896249 100644 --- a/docs/build/reference/transformer/Excel/Excel_ROMAN.md +++ b/docs/build/reference/transformer/Excel/Excel_ROMAN.md @@ -2,10 +2,12 @@ title: "Roman" description: "Excel ROMAN(number; mode): Converts a number into a Roman numeral. The value range must be between 0 and 3999; the modes can be integers from 0 to 4. Number is the number that is to be converted into a Roman numeral. Mode (optional) indicates the degree of simplification. The higher the value, the greater is the simplification of the Roman numeral." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Roman + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_ROUND.md b/docs/build/reference/transformer/Excel/Excel_ROUND.md index 1b569731..aed1fe7e 100644 --- a/docs/build/reference/transformer/Excel/Excel_ROUND.md +++ b/docs/build/reference/transformer/Excel/Excel_ROUND.md @@ -2,10 +2,12 @@ title: "Round" description: "Excel ROUND(number; count): Rounds the given number to a certain number of decimal places according to valid mathematical criteria. Count (optional) is the number of the places to which the value is to be rounded. If the count parameter is negative, only the whole number portion is rounded. It is rounded to the place indicated by the count." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Round + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_ROUNDDOWN.md b/docs/build/reference/transformer/Excel/Excel_ROUNDDOWN.md index c6fe8a73..3eccbb32 100644 --- a/docs/build/reference/transformer/Excel/Excel_ROUNDDOWN.md +++ b/docs/build/reference/transformer/Excel/Excel_ROUNDDOWN.md @@ -2,10 +2,12 @@ title: "Rounddown" description: "Excel ROUNDDOWN(number; count): Rounds the given number. Count (optional) is the number of digits to be rounded down to. If the count parameter is negative, only the whole number portion is rounded. It is rounded to the place indicated by the count." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Rounddown + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_ROUNDUP.md b/docs/build/reference/transformer/Excel/Excel_ROUNDUP.md index b31f3279..a2f03197 100644 --- a/docs/build/reference/transformer/Excel/Excel_ROUNDUP.md +++ b/docs/build/reference/transformer/Excel/Excel_ROUNDUP.md @@ -2,10 +2,12 @@ title: "Roundup" description: "Excel ROUNDUP(number; count): Rounds the given number up. Count (optional) is the number of digits to which rounding up is to be done. If the count parameter is negative, only the whole number portion is rounded. It is rounded to the place indicated by the count." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Roundup + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_SEARCH.md b/docs/build/reference/transformer/Excel/Excel_SEARCH.md index 609a6f12..60f283af 100644 --- a/docs/build/reference/transformer/Excel/Excel_SEARCH.md +++ b/docs/build/reference/transformer/Excel/Excel_SEARCH.md @@ -2,10 +2,12 @@ title: "Search" description: "Excel SEARCH(find_text; text; position): Returns the position of a text segment within a character string. The start of the search can be set as an option. The search text can be a number or any sequence of characters. The search is not case-sensitive. The search supports regular expressions. Find_text is the text to be searched for. Text is the text where the search will take place. Position (optional) is the position in the text where the search is to start." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Search + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_SIGN.md b/docs/build/reference/transformer/Excel/Excel_SIGN.md index 9332ef7c..9910d91d 100644 --- a/docs/build/reference/transformer/Excel/Excel_SIGN.md +++ b/docs/build/reference/transformer/Excel/Excel_SIGN.md @@ -2,10 +2,12 @@ title: "Sign" description: "Excel SIGN(number): Returns the sign of the given number. The function returns the result 1 for a positive sign, -1 for a negative sign, and 0 for zero." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Sign + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_SIN.md b/docs/build/reference/transformer/Excel/Excel_SIN.md index 00b046df..4894ddcd 100644 --- a/docs/build/reference/transformer/Excel/Excel_SIN.md +++ b/docs/build/reference/transformer/Excel/Excel_SIN.md @@ -2,10 +2,12 @@ title: "Sin" description: "Excel SIN(number): Returns the sine of the given number (angle in radians)." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Sin + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_SINH.md b/docs/build/reference/transformer/Excel/Excel_SINH.md index a3f12eaf..c1d5ce18 100644 --- a/docs/build/reference/transformer/Excel/Excel_SINH.md +++ b/docs/build/reference/transformer/Excel/Excel_SINH.md @@ -2,10 +2,12 @@ title: "Sinh" description: "Excel SINH(number): Returns the hyperbolic sine of the given number (angle in radians)." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Sinh + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_SLOPE.md b/docs/build/reference/transformer/Excel/Excel_SLOPE.md index d9b3b95f..526423ff 100644 --- a/docs/build/reference/transformer/Excel/Excel_SLOPE.md +++ b/docs/build/reference/transformer/Excel/Excel_SLOPE.md @@ -2,10 +2,12 @@ title: "Slope" description: "Excel SLOPE(data_Y; data_X): Returns the slope of the linear regression line. Data_Y is the array or matrix of Y data. Data_X is the array or matrix of X data." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Slope + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_SMALL.md b/docs/build/reference/transformer/Excel/Excel_SMALL.md index 9635e018..e5519895 100644 --- a/docs/build/reference/transformer/Excel/Excel_SMALL.md +++ b/docs/build/reference/transformer/Excel/Excel_SMALL.md @@ -2,10 +2,12 @@ title: "Small" description: "Excel SMALL(data; rank_c): Returns the Rank_c-th smallest value in a data set. Data is the cell range of data. Rank_c is the rank of the value (2nd smallest, 3rd smallest, etc.) written as an integer." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Small + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_SQRT.md b/docs/build/reference/transformer/Excel/Excel_SQRT.md index ecf893ac..ee3ae05d 100644 --- a/docs/build/reference/transformer/Excel/Excel_SQRT.md +++ b/docs/build/reference/transformer/Excel/Excel_SQRT.md @@ -2,10 +2,12 @@ title: "Sqrt" description: "Excel SQRT(number): Returns the positive square root of the given number. The value of the number must be positive." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Sqrt + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_STANDARDIZE.md b/docs/build/reference/transformer/Excel/Excel_STANDARDIZE.md index 6bc3e123..6026a30b 100644 --- a/docs/build/reference/transformer/Excel/Excel_STANDARDIZE.md +++ b/docs/build/reference/transformer/Excel/Excel_STANDARDIZE.md @@ -2,10 +2,12 @@ title: "Standardize" description: "Excel STANDARDIZE(number; mean; STDEV): Converts a random variable to a normalized value. Number is the value to be standardized. Mean is the arithmetic mean of the distribution. STDEV is the standard deviation of the distribution." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Standardize + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_STDEV.md b/docs/build/reference/transformer/Excel/Excel_STDEV.md index edc96e0f..140e7eba 100644 --- a/docs/build/reference/transformer/Excel/Excel_STDEV.md +++ b/docs/build/reference/transformer/Excel/Excel_STDEV.md @@ -2,10 +2,12 @@ title: "Stdev" description: "Excel STDEV(number_1; number_2; ... number_30): Estimates the standard deviation based on a sample. Number_1; number_2; ... number_30 are numerical values or ranges representing a sample based on an entire population." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Stdev + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_STDEVA.md b/docs/build/reference/transformer/Excel/Excel_STDEVA.md index 8eccefa8..2ed89cf9 100644 --- a/docs/build/reference/transformer/Excel/Excel_STDEVA.md +++ b/docs/build/reference/transformer/Excel/Excel_STDEVA.md @@ -2,10 +2,12 @@ title: "Stdeva" description: "Excel STDEVA(value_1; value_2; ... value_30): Calculates the standard deviation of an estimation based on a sample. Value_1; value_2; ... value_30 are values or ranges representing a sample derived from an entire population. Text has the value 0." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Stdeva + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_STDEVP.md b/docs/build/reference/transformer/Excel/Excel_STDEVP.md index 48a85453..e883ae69 100644 --- a/docs/build/reference/transformer/Excel/Excel_STDEVP.md +++ b/docs/build/reference/transformer/Excel/Excel_STDEVP.md @@ -2,10 +2,12 @@ title: "Stdevp" description: "Excel STDEVP(number_1; number_2; ... number_30): Calculates the standard deviation based on the entire population. Number_1; number_2; ... number_30 are numerical values or ranges representing a sample based on an entire population." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Stdevp + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_STDEVPA.md b/docs/build/reference/transformer/Excel/Excel_STDEVPA.md index e1077c2c..6d665f2c 100644 --- a/docs/build/reference/transformer/Excel/Excel_STDEVPA.md +++ b/docs/build/reference/transformer/Excel/Excel_STDEVPA.md @@ -2,10 +2,12 @@ title: "Stdevpa" description: "Excel STDEVPA(value_1; value_2; ... value_30): Calculates the standard deviation based on the entire population. Value_1; value_2; ... value_30 are values or ranges representing a sample derived from an entire population. Text has the value 0." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Stdevpa + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_SUBSTITUTE.md b/docs/build/reference/transformer/Excel/Excel_SUBSTITUTE.md index 1bb656ea..36496ba4 100644 --- a/docs/build/reference/transformer/Excel/Excel_SUBSTITUTE.md +++ b/docs/build/reference/transformer/Excel/Excel_SUBSTITUTE.md @@ -2,10 +2,12 @@ title: "Substitute" description: "Excel SUBSTITUTE(text; search_text; new text; occurrence): Substitutes new text for old text in a string. Text is the text in which text segments are to be exchanged. Search_text is the text segment that is to be replaced (a number of times). New text is the text that is to replace the text segment. Occurrence (optional) indicates how many occurrences of the search text are to be replaced. If this parameter is missing, the search text is replaced throughout." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Substitute + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_SUM.md b/docs/build/reference/transformer/Excel/Excel_SUM.md index 0d93f3f1..625e544f 100644 --- a/docs/build/reference/transformer/Excel/Excel_SUM.md +++ b/docs/build/reference/transformer/Excel/Excel_SUM.md @@ -2,10 +2,12 @@ title: "Sum" description: "Excel SUM(number_1; number_2; ... number_30): Adds all the numbers in a range of cells. Number_1; number_2;... number_30 are up to 30 arguments whose sum is to be calculated. You can also enter a range using cell references." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Sum + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_SUMPRODUCT.md b/docs/build/reference/transformer/Excel/Excel_SUMPRODUCT.md index 21a73263..33e0f05c 100644 --- a/docs/build/reference/transformer/Excel/Excel_SUMPRODUCT.md +++ b/docs/build/reference/transformer/Excel/Excel_SUMPRODUCT.md @@ -2,10 +2,12 @@ title: "Sumproduct" description: "Excel SUMPRODUCT(array 1; array 2; ...array 30): Multiplies corresponding elements in the given arrays, and returns the sum of those products. Array 1; array 2;...array 30 are arrays whose corresponding elements are to be multiplied. At least one array must be part of the argument list. If only one array is given, all array elements are summed." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Sumproduct + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_SUMSQ.md b/docs/build/reference/transformer/Excel/Excel_SUMSQ.md index c5666030..0f62562d 100644 --- a/docs/build/reference/transformer/Excel/Excel_SUMSQ.md +++ b/docs/build/reference/transformer/Excel/Excel_SUMSQ.md @@ -2,10 +2,12 @@ title: "Sumsq" description: "Excel SUMSQ(number_1; number_2; ... number_30): Calculates the sum of the squares of numbers (totaling up of the squares of the arguments) Number_1; number_2;... number_30 are up to 30 arguments, the sum of whose squares is to be calculated." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Sumsq + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_SUMX2MY2.md b/docs/build/reference/transformer/Excel/Excel_SUMX2MY2.md index 67c0be73..8b09ec88 100644 --- a/docs/build/reference/transformer/Excel/Excel_SUMX2MY2.md +++ b/docs/build/reference/transformer/Excel/Excel_SUMX2MY2.md @@ -2,10 +2,12 @@ title: "Sumx2my2" description: "Excel SUMX2MY2(array_X; array_Y): Returns the sum of the difference of squares of corresponding values in two arrays. Array_X is the first array whose elements are to be squared and added. Array_Y is the second array whose elements are to be squared and subtracted." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Sumx2my2 + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_SUMX2PY2.md b/docs/build/reference/transformer/Excel/Excel_SUMX2PY2.md index 0fa4baa2..9744d112 100644 --- a/docs/build/reference/transformer/Excel/Excel_SUMX2PY2.md +++ b/docs/build/reference/transformer/Excel/Excel_SUMX2PY2.md @@ -2,10 +2,12 @@ title: "Sumx2py2" description: "Excel SUMX2PY2(array_X; array_Y): Returns the sum of the sum of squares of corresponding values in two arrays. Array_X is the first array whose arguments are to be squared and added. Array_Y is the second array, whose elements are to be added and squared." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Sumx2py2 + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_SUMXMY2.md b/docs/build/reference/transformer/Excel/Excel_SUMXMY2.md index f20c6f01..0bcd1fb8 100644 --- a/docs/build/reference/transformer/Excel/Excel_SUMXMY2.md +++ b/docs/build/reference/transformer/Excel/Excel_SUMXMY2.md @@ -2,10 +2,12 @@ title: "Sumxmy2" description: "Excel SUMXMY2(array_X; array_Y): Adds the squares of the variance between corresponding values in two arrays. Array_X is the first array whose elements are to be subtracted and squared. Array_Y is the second array, whose elements are to be subtracted and squared." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Sumxmy2 + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_TAN.md b/docs/build/reference/transformer/Excel/Excel_TAN.md index ba61e33c..bd96c2d7 100644 --- a/docs/build/reference/transformer/Excel/Excel_TAN.md +++ b/docs/build/reference/transformer/Excel/Excel_TAN.md @@ -2,10 +2,12 @@ title: "Tan" description: "Excel TAN(number): Returns the tangent of the given number (angle in radians)." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Tan + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_TANH.md b/docs/build/reference/transformer/Excel/Excel_TANH.md index 412f90e7..5c14c1af 100644 --- a/docs/build/reference/transformer/Excel/Excel_TANH.md +++ b/docs/build/reference/transformer/Excel/Excel_TANH.md @@ -2,10 +2,12 @@ title: "Tanh" description: "Excel TANH(number): Returns the hyperbolic tangent of the given number (angle in radians)." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Tanh + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_TDIST.md b/docs/build/reference/transformer/Excel/Excel_TDIST.md index 21589823..8418830c 100644 --- a/docs/build/reference/transformer/Excel/Excel_TDIST.md +++ b/docs/build/reference/transformer/Excel/Excel_TDIST.md @@ -2,10 +2,12 @@ title: "Tdist" description: "Excel TDIST(number; degrees_freedom; mode): Returns the t-distribution for the given Number. Degrees_freedom is the number of degrees of freedom for the t-distribution. Mode = 1 returns the one-tailed test, Mode = 2 returns the two-tailed test." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Tdist + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_TRUE.md b/docs/build/reference/transformer/Excel/Excel_TRUE.md index a2041362..2bd1847d 100644 --- a/docs/build/reference/transformer/Excel/Excel_TRUE.md +++ b/docs/build/reference/transformer/Excel/Excel_TRUE.md @@ -2,10 +2,12 @@ title: "True" description: "Excel TRUE(): Sets the logical value to TRUE. The TRUE() function does not require any arguments." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # True + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_TRUNC.md b/docs/build/reference/transformer/Excel/Excel_TRUNC.md index 95b341f4..6934e857 100644 --- a/docs/build/reference/transformer/Excel/Excel_TRUNC.md +++ b/docs/build/reference/transformer/Excel/Excel_TRUNC.md @@ -2,10 +2,12 @@ title: "Trunc" description: "Excel TRUNC(number; count): Truncates a number to an integer by removing the fractional part of the number according to the precision specified in Tools > Options > OpenOffice.org Calc > Calculate. Number is the number whose decimal places are to be cut off. Count is the number of decimal places which are not cut off." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Trunc + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_VAR.md b/docs/build/reference/transformer/Excel/Excel_VAR.md index ebac4d6a..64138864 100644 --- a/docs/build/reference/transformer/Excel/Excel_VAR.md +++ b/docs/build/reference/transformer/Excel/Excel_VAR.md @@ -2,10 +2,12 @@ title: "Var" description: "Excel VAR(number_1; number_2; ... number_30): Estimates the variance based on a sample. Number_1; number_2; ... number_30 are numerical values or ranges representing a sample based on an entire population." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Var + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_VARA.md b/docs/build/reference/transformer/Excel/Excel_VARA.md index ebb5d43b..cb0e8c82 100644 --- a/docs/build/reference/transformer/Excel/Excel_VARA.md +++ b/docs/build/reference/transformer/Excel/Excel_VARA.md @@ -2,10 +2,12 @@ title: "Vara" description: "Excel VARA(value_1; value_2; ... value_30): Estimates a variance based on a sample. The value of text is 0. Value_1; value_2; ... value_30 are values or ranges representing a sample derived from an entire population. Text has the value 0." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Vara + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_VARP.md b/docs/build/reference/transformer/Excel/Excel_VARP.md index fedbc893..342e3953 100644 --- a/docs/build/reference/transformer/Excel/Excel_VARP.md +++ b/docs/build/reference/transformer/Excel/Excel_VARP.md @@ -2,10 +2,12 @@ title: "Varp" description: "Excel VARP(Number_1; number_2; ... number_30): Calculates a variance based on the entire population. Number_1; number_2; ... number_30 are numerical values or ranges representing an entire population." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Varp + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Excel/Excel_VARPA.md b/docs/build/reference/transformer/Excel/Excel_VARPA.md index d06ec5c7..59f0caa5 100644 --- a/docs/build/reference/transformer/Excel/Excel_VARPA.md +++ b/docs/build/reference/transformer/Excel/Excel_VARPA.md @@ -2,10 +2,12 @@ title: "Varpa" description: "Excel VARPA(value_1; value_2; .. .value_30): Calculates the variance based on the entire population. The value of text is 0. Value_1; value_2; ... value_30 are values or ranges representing an entire population." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Varpa + @@ -28,4 +30,4 @@ The name of the Excel function ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Extract/regexExtract.md b/docs/build/reference/transformer/Extract/regexExtract.md index 7b7f6e97..4d9583e1 100644 --- a/docs/build/reference/transformer/Extract/regexExtract.md +++ b/docs/build/reference/transformer/Extract/regexExtract.md @@ -2,10 +2,12 @@ title: "Regex extract" description: "Extracts one or all matches of a regular expression within the input. If the regular expression contains one or more capturing groups, only the first group will be considered." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Regex extract + @@ -152,9 +154,9 @@ A compilation of the available constructs for building regular expressions is av Regular expression -- ID: `regex` -- Datatype: `string` -- Default Value: `None` +* ID: `regex` +* Datatype: `string` +* Default Value: `None` @@ -162,9 +164,9 @@ Regular expression If true, all matches are extracted. If false, only the first match is extracted (default). -- ID: `extractAll` -- Datatype: `boolean` -- Default Value: `false` +* ID: `extractAll` +* Datatype: `boolean` +* Default Value: `false` @@ -172,4 +174,4 @@ If true, all matches are extracted. If false, only the first match is extracted ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Filter/filterByLength.md b/docs/build/reference/transformer/Filter/filterByLength.md index fe05f36b..e2938e6d 100644 --- a/docs/build/reference/transformer/Filter/filterByLength.md +++ b/docs/build/reference/transformer/Filter/filterByLength.md @@ -2,10 +2,12 @@ title: "Filter by length" description: "Removes all strings that are shorter than 'min' characters and longer than 'max' characters." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Filter by length + @@ -39,4 +41,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Filter/filterByRegex.md b/docs/build/reference/transformer/Filter/filterByRegex.md index 16ed8664..6c8ff4c7 100644 --- a/docs/build/reference/transformer/Filter/filterByRegex.md +++ b/docs/build/reference/transformer/Filter/filterByRegex.md @@ -2,10 +2,12 @@ title: "Filter by regex" description: "Removes all strings that do NOT match a regex. If 'negate' is true, only strings will be removed that match the regex." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Filter by regex + @@ -39,4 +41,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Filter/removeDefaultStopWords.md b/docs/build/reference/transformer/Filter/removeDefaultStopWords.md index 69296745..a2d7ce5c 100644 --- a/docs/build/reference/transformer/Filter/removeDefaultStopWords.md +++ b/docs/build/reference/transformer/Filter/removeDefaultStopWords.md @@ -2,10 +2,12 @@ title: "Remove default stop words" description: "Removes stop words based on a default stop word list." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Remove default stop words + @@ -50,4 +52,4 @@ If a different stop word list is needed, the Remove stop words plugin supports p ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Filter/removeEmptyValues.md b/docs/build/reference/transformer/Filter/removeEmptyValues.md index 28cc7f9a..125dc577 100644 --- a/docs/build/reference/transformer/Filter/removeEmptyValues.md +++ b/docs/build/reference/transformer/Filter/removeEmptyValues.md @@ -2,10 +2,12 @@ title: "Remove empty values" description: "Removes empty values." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Remove empty values + @@ -42,4 +44,4 @@ Removes empty values. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Filter/removeRemoteStopWords.md b/docs/build/reference/transformer/Filter/removeRemoteStopWords.md index 4a1cf6ab..cec59baa 100644 --- a/docs/build/reference/transformer/Filter/removeRemoteStopWords.md +++ b/docs/build/reference/transformer/Filter/removeRemoteStopWords.md @@ -2,10 +2,12 @@ title: "Remove remote stop words" description: "Removes stop words based on a stop word list remote URL." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Remove remote stop words + @@ -33,19 +35,19 @@ This separator is therefore part of the semantic contract: it determines what ev --- **Example 1:** -* Input values: +- Input values: 1. `[To be or not to be, that is the question]` -* Returns: `[To, question]` +- Returns: `[To, question]` --- **Example 2:** -* Input values: +- Input values: 1. `[It always seems impossible, until it's done]` -* Returns: `[It impossible, ]` +- Returns: `[It impossible, ]` @@ -76,4 +78,4 @@ RegEx for detecting words ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Filter/removeStopWords.md b/docs/build/reference/transformer/Filter/removeStopWords.md index 097d8042..edb926a5 100644 --- a/docs/build/reference/transformer/Filter/removeStopWords.md +++ b/docs/build/reference/transformer/Filter/removeStopWords.md @@ -2,10 +2,12 @@ title: "Remove stop words" description: "Removes stop words based on a stop word list resource." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Remove stop words + @@ -56,9 +58,9 @@ Additionally, notice the simpler filter 'removeDefaultStopWords', which uses a d Resource for the stop word list -- ID: `stopWordList` -- Datatype: `resource` -- Default Value: `None` +* ID: `stopWordList` +* Datatype: `resource` +* Default Value: `None` @@ -66,9 +68,9 @@ Resource for the stop word list RegEx for detecting words -- ID: `separator` -- Datatype: `string` -- Default Value: `[\s-]+` +* ID: `separator` +* Datatype: `string` +* Default Value: `[\s-]+` @@ -76,4 +78,4 @@ RegEx for detecting words ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Filter/removeValues.md b/docs/build/reference/transformer/Filter/removeValues.md index f81f8fd7..2ba70ba8 100644 --- a/docs/build/reference/transformer/Filter/removeValues.md +++ b/docs/build/reference/transformer/Filter/removeValues.md @@ -2,10 +2,12 @@ title: "Remove values" description: "Removes values that contain words from a blacklist. The blacklist values are separated with commas." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Remove values + @@ -29,4 +31,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Geo/RetrieveCoordinates.md b/docs/build/reference/transformer/Geo/RetrieveCoordinates.md index 97c573dd..8a87e4f3 100644 --- a/docs/build/reference/transformer/Geo/RetrieveCoordinates.md +++ b/docs/build/reference/transformer/Geo/RetrieveCoordinates.md @@ -2,10 +2,12 @@ title: "Retrieve coordinates" description: "Retrieves geographic coordinates using Nominatim." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Retrieve coordinates + @@ -21,18 +23,18 @@ The default configuration is as follows: # url = "https://nominatim.eccenca.com/search" url = "https://photon.komoot.de/api" # url = https://api-adresse.data.gouv.fr/search - + # Additional URL parameters to be attached to all HTTP search requests. Example: '&countrycodes=de&addressdetails=1'. # Will be attached in addition to the parameters set on each search operator directly. searchParameters = "" - + # The minimum pause time between subsequent queries pauseTime = 1s - + # Number of coordinates to be cached in-memory cacheSize = 10 } - + In general, all services adhering to the [Nominatim search API](https://nominatim.org/release-docs/develop/api/Search/) should be usable. Please note that when using public services, the pause time should be set to avoid overloading. @@ -60,4 +62,3 @@ Additional URL parameters to be attached to each HTTP search request. Example: ' - Default Value: `None` - diff --git a/docs/build/reference/transformer/Geo/RetrieveLatitude.md b/docs/build/reference/transformer/Geo/RetrieveLatitude.md index 777d2a07..87183c92 100644 --- a/docs/build/reference/transformer/Geo/RetrieveLatitude.md +++ b/docs/build/reference/transformer/Geo/RetrieveLatitude.md @@ -2,10 +2,12 @@ title: "Retrieve latitude" description: "Retrieves geographic coordinates using Nominatim and returns the latitude." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Retrieve latitude + @@ -21,18 +23,18 @@ The default configuration is as follows: # url = "https://nominatim.eccenca.com/search" url = "https://photon.komoot.de/api" # url = https://api-adresse.data.gouv.fr/search - + # Additional URL parameters to be attached to all HTTP search requests. Example: '&countrycodes=de&addressdetails=1'. # Will be attached in addition to the parameters set on each search operator directly. searchParameters = "" - + # The minimum pause time between subsequent queries pauseTime = 1s - + # Number of coordinates to be cached in-memory cacheSize = 10 } - + In general, all services adhering to the [Nominatim search API](https://nominatim.org/release-docs/develop/api/Search/) should be usable. Please note that when using public services, the pause time should be set to avoid overloading. @@ -60,4 +62,3 @@ Additional URL parameters to be attached to each HTTP search request. Example: ' - Default Value: `None` - diff --git a/docs/build/reference/transformer/Geo/RetrieveLongitude.md b/docs/build/reference/transformer/Geo/RetrieveLongitude.md index 210a2e51..f90cdc0b 100644 --- a/docs/build/reference/transformer/Geo/RetrieveLongitude.md +++ b/docs/build/reference/transformer/Geo/RetrieveLongitude.md @@ -2,10 +2,12 @@ title: "Retrieve longitude" description: "Retrieves geographic coordinates using Nominatim and returns the longitude." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Retrieve longitude + @@ -21,18 +23,18 @@ The default configuration is as follows: # url = "https://nominatim.eccenca.com/search" url = "https://photon.komoot.de/api" # url = https://api-adresse.data.gouv.fr/search - + # Additional URL parameters to be attached to all HTTP search requests. Example: '&countrycodes=de&addressdetails=1'. # Will be attached in addition to the parameters set on each search operator directly. searchParameters = "" - + # The minimum pause time between subsequent queries pauseTime = 1s - + # Number of coordinates to be cached in-memory cacheSize = 10 } - + In general, all services adhering to the [Nominatim search API](https://nominatim.org/release-docs/develop/api/Search/) should be usable. Please note that when using public services, the pause time should be set to avoid overloading. @@ -60,4 +62,3 @@ Additional URL parameters to be attached to each HTTP search request. Example: ' - Default Value: `None` - diff --git a/docs/build/reference/transformer/Linguistic/NYSIIS.md b/docs/build/reference/transformer/Linguistic/NYSIIS.md index c931748a..c5f19ff6 100644 --- a/docs/build/reference/transformer/Linguistic/NYSIIS.md +++ b/docs/build/reference/transformer/Linguistic/NYSIIS.md @@ -2,10 +2,12 @@ title: "NYSIIS" description: "NYSIIS phonetic encoding." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # NYSIIS + @@ -30,12 +32,12 @@ necessarily contain this limitation. This plugin doesn't. ### Modified NYSIIS The **Modified NYSIIS** is an improvement of the NYSIIS algorithm. Its working is illustrated, step by step, in -http://www.dropby.com/NYSIIS.html. +. ## Examples We can get an idea of the output of the NYSIIS algorithm using an online version of it such as the already mentioned -http://www.dropby.com/NYSIIS.html. It contains both the (plain) NYSIIS and the _modified_ NYSIIS algorithms. +. It contains both the (plain) NYSIIS and the _modified_ NYSIIS algorithms. As a comparison of the two versions of NYSIIS, we give a few examples: @@ -51,9 +53,9 @@ As a comparison of the two versions of NYSIIS, we give a few examples: No description -- ID: `refined` -- Datatype: `boolean` -- Default Value: `true` +* ID: `refined` +* Datatype: `boolean` +* Default Value: `true` @@ -61,4 +63,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Linguistic/metaphone.md b/docs/build/reference/transformer/Linguistic/metaphone.md index ff91fdee..6420d805 100644 --- a/docs/build/reference/transformer/Linguistic/metaphone.md +++ b/docs/build/reference/transformer/Linguistic/metaphone.md @@ -2,10 +2,12 @@ title: "Metaphone" description: "Metaphone phonetic encoding." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Metaphone + @@ -41,4 +43,4 @@ Illustrative examples: ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Linguistic/normalizeChars.md b/docs/build/reference/transformer/Linguistic/normalizeChars.md index 4effbc86..1953573a 100644 --- a/docs/build/reference/transformer/Linguistic/normalizeChars.md +++ b/docs/build/reference/transformer/Linguistic/normalizeChars.md @@ -2,10 +2,12 @@ title: "Normalize chars" description: "Replaces diacritical characters with non-diacritical ones (eg, ö -> o), plus some specialities like transforming æ -> ae, ß -> ss." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Normalize chars + @@ -19,4 +21,4 @@ Replaces diacritical characters with non-diacritical ones (eg, ö -> o), plus so ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Linguistic/soundex.md b/docs/build/reference/transformer/Linguistic/soundex.md index 059f9021..ba455216 100644 --- a/docs/build/reference/transformer/Linguistic/soundex.md +++ b/docs/build/reference/transformer/Linguistic/soundex.md @@ -2,10 +2,12 @@ title: "Soundex" description: "Soundex algorithm." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Soundex + @@ -58,7 +60,7 @@ used by setting the plugin parameter `refined` to `true` (default). Its mapping ### Soundex We can get an idea of the output of the Soundex algorithm using an online Soundex Converter such as -https://www.mainegenealogy.net/soundex_converter.asp. +. * `robert` and `rupert` lead to the same Soundex index: `r163`. * `euler` leads to `e460`, `gauss` is `g200` and `hilbert` corresponds to `h416`. @@ -68,7 +70,7 @@ https://www.mainegenealogy.net/soundex_converter.asp. * `braz` and `broz` lead to the same Refined Soundex index: `b1905`. * `caren`, `carren`, `coram`, `corran`, `curreen` and `curwen` are all encoded with `c30908`. * `hairs`, `hark`, `hars`, `hayers`, `heers` and `hiers` are all mapped to `h093`. -* All sorts of variations of `lambard`, such as `lambart`, `lambert`, `lambird` or `lampaert`, lead to `l7081096`. +* All sorts of variations of `lambard`, such as `lambart`, `lambert`, `lambird` or `lampaert`, lead to `l7081096`. ## Parameter @@ -77,9 +79,9 @@ https://www.mainegenealogy.net/soundex_converter.asp. No description -- ID: `refined` -- Datatype: `boolean` -- Default Value: `true` +* ID: `refined` +* Datatype: `boolean` +* Default Value: `true` @@ -87,4 +89,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Linguistic/stem.md b/docs/build/reference/transformer/Linguistic/stem.md index 75940976..4d1237cf 100644 --- a/docs/build/reference/transformer/Linguistic/stem.md +++ b/docs/build/reference/transformer/Linguistic/stem.md @@ -2,10 +2,12 @@ title: "Stem" description: "Stems a string using the Porter Stemmer." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Stem + @@ -19,4 +21,4 @@ Stems a string using the Porter Stemmer. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Metadata/fileHash.md b/docs/build/reference/transformer/Metadata/fileHash.md index e643f866..43cba6ef 100644 --- a/docs/build/reference/transformer/Metadata/fileHash.md +++ b/docs/build/reference/transformer/Metadata/fileHash.md @@ -2,10 +2,12 @@ title: "File hash" description: "Calculates the hash sum of a file. The hash sum is cached so that subsequent calls to this operator are fast. Note that initially and every time the specified resource has been updated, this operator might take a long time (depending on the file size). This operator supports using different hash algorithms from the [Secure Hash Algorithms family](https://en.wikipedia.org/wiki/Secure_Hash_Algorithms) (SHA, e.g. SHA256) and two algorithms from the [Message-Digest Algorithm family](https://en.wikipedia.org/wiki/MD5) (MD2 / MD5). Please be aware that some of these algorithms are not secure regarding collision- and other attacks. Note: This transform operator ignores any inputs." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # File hash + @@ -42,4 +44,4 @@ The hash algorithm to be used. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Metadata/inputFileAttributes.md b/docs/build/reference/transformer/Metadata/inputFileAttributes.md index 4e141581..73149e20 100644 --- a/docs/build/reference/transformer/Metadata/inputFileAttributes.md +++ b/docs/build/reference/transformer/Metadata/inputFileAttributes.md @@ -2,10 +2,12 @@ title: "Input file attributes" description: "Retrieves a metadata attribute from the input file (such as the file name)." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Input file attributes + @@ -29,4 +31,4 @@ File attribute to be retrieved from the input dataset. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Metadata/inputTaskAttributes.md b/docs/build/reference/transformer/Metadata/inputTaskAttributes.md index 09faa10d..165e1f34 100644 --- a/docs/build/reference/transformer/Metadata/inputTaskAttributes.md +++ b/docs/build/reference/transformer/Metadata/inputTaskAttributes.md @@ -2,10 +2,12 @@ title: "Input task attributes" description: "Retrieves individual attributes from the input task (such as the modified date) or the entire task as JSON." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Input task attributes + @@ -29,4 +31,4 @@ Path to retrieve from the JSON, such as 'metadata/modified'. If left empty, the ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Normalize/alphaReduce.md b/docs/build/reference/transformer/Normalize/alphaReduce.md index fd4fa278..ad941618 100644 --- a/docs/build/reference/transformer/Normalize/alphaReduce.md +++ b/docs/build/reference/transformer/Normalize/alphaReduce.md @@ -2,10 +2,12 @@ title: "Strip non-alphabetic characters" description: "Strips all non-alphabetic characters from a string. Spaces are retained." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Strip non-alphabetic characters + @@ -19,4 +21,4 @@ Strips all non-alphabetic characters from a string. Spaces are retained. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Normalize/camelCase.md b/docs/build/reference/transformer/Normalize/camelCase.md index d6c5ce42..a2c6483b 100644 --- a/docs/build/reference/transformer/Normalize/camelCase.md +++ b/docs/build/reference/transformer/Normalize/camelCase.md @@ -2,10 +2,12 @@ title: "Camel case" description: "Converts a string to camel case. Upper camel case is the default, lower camel case can be chosen." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Camel case + @@ -96,9 +98,9 @@ Converts a string to camel case. Upper camel case is the default, lower camel ca If true, lower camel case (aka. dromedary case) is used, otherwise upper camel case is used. -- ID: `isDromedary` -- Datatype: `boolean` -- Default Value: `false` +* ID: `isDromedary` +* Datatype: `boolean` +* Default Value: `false` @@ -106,4 +108,4 @@ If true, lower camel case (aka. dromedary case) is used, otherwise upper camel c ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Normalize/capitalize.md b/docs/build/reference/transformer/Normalize/capitalize.md index 91e51282..bc515431 100644 --- a/docs/build/reference/transformer/Normalize/capitalize.md +++ b/docs/build/reference/transformer/Normalize/capitalize.md @@ -2,10 +2,12 @@ title: "Capitalize" description: "Capitalizes the string i.e. converts the first character to upper case. If 'allWords' is set to true, all words are capitalized and not only the first character." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Capitalize + @@ -48,9 +50,9 @@ Capitalizes the string i.e. converts the first character to upper case. If 'allW No description -- ID: `allWords` -- Datatype: `boolean` -- Default Value: `false` +* ID: `allWords` +* Datatype: `boolean` +* Default Value: `false` @@ -58,4 +60,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Normalize/htmlCleaner.md b/docs/build/reference/transformer/Normalize/htmlCleaner.md index be1dda48..851dd16c 100644 --- a/docs/build/reference/transformer/Normalize/htmlCleaner.md +++ b/docs/build/reference/transformer/Normalize/htmlCleaner.md @@ -2,10 +2,12 @@ title: "Clean HTML" description: "Cleans HTML markup using a tag whitelist and allows selection of HTML sections with XPath or CSS selector expressions." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Clean HTML + @@ -73,4 +75,4 @@ Use defaults for empty tag and attribute whitelists. If the attribute while list ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Normalize/lowerCase.md b/docs/build/reference/transformer/Normalize/lowerCase.md index 84a91988..a8f39d11 100644 --- a/docs/build/reference/transformer/Normalize/lowerCase.md +++ b/docs/build/reference/transformer/Normalize/lowerCase.md @@ -2,10 +2,12 @@ title: "Lower case" description: "Converts a string to lower case." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Lower case + @@ -33,4 +35,4 @@ Converts a string to lower case. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Normalize/removeBlanks.md b/docs/build/reference/transformer/Normalize/removeBlanks.md index 1d364e1e..0174b87b 100644 --- a/docs/build/reference/transformer/Normalize/removeBlanks.md +++ b/docs/build/reference/transformer/Normalize/removeBlanks.md @@ -2,10 +2,12 @@ title: "Remove blanks" description: "Remove whitespace from a string." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Remove blanks + @@ -19,4 +21,4 @@ Remove whitespace from a string. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Normalize/removeDuplicates.md b/docs/build/reference/transformer/Normalize/removeDuplicates.md index f858e9da..33e100a6 100644 --- a/docs/build/reference/transformer/Normalize/removeDuplicates.md +++ b/docs/build/reference/transformer/Normalize/removeDuplicates.md @@ -2,10 +2,12 @@ title: "Remove duplicates" description: "Removes duplicated values, making a value sequence distinct." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Remove duplicates + @@ -19,4 +21,4 @@ Removes duplicated values, making a value sequence distinct. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Normalize/removeParentheses.md b/docs/build/reference/transformer/Normalize/removeParentheses.md index f730586b..a17f7f43 100644 --- a/docs/build/reference/transformer/Normalize/removeParentheses.md +++ b/docs/build/reference/transformer/Normalize/removeParentheses.md @@ -2,10 +2,12 @@ title: "Remove parentheses" description: "Remove all parentheses including their content, e.g., transforms 'Berlin (City)' -> 'Berlin'." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Remove parentheses + @@ -19,4 +21,4 @@ Remove all parentheses including their content, e.g., transforms 'Berlin (City)' ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Normalize/removeSpecialChars.md b/docs/build/reference/transformer/Normalize/removeSpecialChars.md index 18c59c6d..e0e237fa 100644 --- a/docs/build/reference/transformer/Normalize/removeSpecialChars.md +++ b/docs/build/reference/transformer/Normalize/removeSpecialChars.md @@ -2,10 +2,12 @@ title: "Remove special chars" description: "Remove special characters (including punctuation) from a string." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Remove special chars + @@ -19,4 +21,4 @@ Remove special characters (including punctuation) from a string. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Normalize/sortWords.md b/docs/build/reference/transformer/Normalize/sortWords.md index 1efac0d5..ac6a5c35 100644 --- a/docs/build/reference/transformer/Normalize/sortWords.md +++ b/docs/build/reference/transformer/Normalize/sortWords.md @@ -2,10 +2,12 @@ title: "Sort words" description: "Sorts all words in each value lexicographically." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Sort words + @@ -19,28 +21,28 @@ Sorts all words in each value lexicographically. --- **Example 1:** -* Input values: +- Input values: 1. `[]` -* Returns: `[]` +- Returns: `[]` --- **Example 2:** -* Input values: +- Input values: 1. `[c a b]` -* Returns: `[a b c]` +- Returns: `[a b c]` --- **Example 3:** -* Input values: +- Input values: 1. `[Hans Hansa Hamburg, München Marburg]` -* Returns: `[Hamburg Hans Hansa, Marburg München]` +- Returns: `[Hamburg Hans Hansa, Marburg München]` @@ -63,7 +65,7 @@ Separator to be inserted between sorted words. - ID: `glue` - Datatype: `string` -- Default Value: ` ` +- Default Value: `` @@ -71,4 +73,4 @@ Separator to be inserted between sorted words. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Normalize/trim.md b/docs/build/reference/transformer/Normalize/trim.md index b28653ea..1a00498c 100644 --- a/docs/build/reference/transformer/Normalize/trim.md +++ b/docs/build/reference/transformer/Normalize/trim.md @@ -2,10 +2,12 @@ title: "Trim" description: "Remove leading and trailing whitespaces." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Trim + @@ -19,4 +21,4 @@ Remove leading and trailing whitespaces. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Normalize/upperCase.md b/docs/build/reference/transformer/Normalize/upperCase.md index c81ff465..e54035b9 100644 --- a/docs/build/reference/transformer/Normalize/upperCase.md +++ b/docs/build/reference/transformer/Normalize/upperCase.md @@ -2,10 +2,12 @@ title: "Upper case" description: "Converts a string to upper case." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Upper case + @@ -19,4 +21,4 @@ Converts a string to upper case. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Normalize/uriFix.md b/docs/build/reference/transformer/Normalize/uriFix.md index aa731ccc..704131dd 100644 --- a/docs/build/reference/transformer/Normalize/uriFix.md +++ b/docs/build/reference/transformer/Normalize/uriFix.md @@ -2,10 +2,12 @@ title: "Fix URI" description: "Generates valid absolute URIs from the given values. Already valid absolute URIs are left untouched." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Fix URI + @@ -114,9 +116,9 @@ Generates valid absolute URIs from the given values. Already valid absolute URIs No description -- ID: `uriPrefix` -- Datatype: `string` -- Default Value: `urn:url-encoded-value:` +* ID: `uriPrefix` +* Datatype: `string` +* Default Value: `urn:url-encoded-value:` @@ -124,4 +126,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Normalize/urlEncode.md b/docs/build/reference/transformer/Normalize/urlEncode.md index cc84618a..0230ee4b 100644 --- a/docs/build/reference/transformer/Normalize/urlEncode.md +++ b/docs/build/reference/transformer/Normalize/urlEncode.md @@ -2,10 +2,12 @@ title: "Encode URL" description: "URL encodes the string." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Encode URL + @@ -55,9 +57,8 @@ URL encodes the string. The character encoding. -- ID: `encoding` -- Datatype: `string` -- Default Value: `UTF-8` - +* ID: `encoding` +* Datatype: `string` +* Default Value: `UTF-8` diff --git a/docs/build/reference/transformer/Numeric/PhysicalQuantitiesNormalizer.md b/docs/build/reference/transformer/Numeric/PhysicalQuantitiesNormalizer.md index 930e1b1d..2a7015b1 100644 --- a/docs/build/reference/transformer/Numeric/PhysicalQuantitiesNormalizer.md +++ b/docs/build/reference/transformer/Numeric/PhysicalQuantitiesNormalizer.md @@ -2,10 +2,12 @@ title: "Normalize physical quantity" description: "Normalizes physical quantities. By default, all quantities are normalized to their base unit (SI), which is overridable. For instance, lengths will be normalized to metres by default." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Normalize physical quantity + @@ -25,6 +27,7 @@ The physical magnitudes are expected to be expressed in the form `{Number}{UnitP Spaces between the number and the physical unit, consisting of unit prefix and symbol, are optional. Additionally: + * If one input is provided, the physical quantities are parsed from the provided strings of the form `"1 km"`. * If two inputs are provided, the numeric values are parsed from the first input and the units from the second input. @@ -36,6 +39,7 @@ Additionally: Time is expressed in seconds (symbol: `s`). The following alternative symbols are supported: + * `mo_s`: day*29.53059 * `mo_g`: year/12.0 * `a`: day*365.25 @@ -53,6 +57,7 @@ The following alternative symbols are supported: Length is expressed in metres (symbol: `m`). The following alternative symbols are supported: + * `in`: c(cm*254.0) * `nmi`: m*1852.0 * `Ao`: dnm @@ -70,6 +75,7 @@ The following alternative symbols are supported: Mass is expressed in kilograms (symbol: `kg`). The following alternative symbols are supported: + * `lb`: lb * `ston`: hlb*20.0 * `t`: Mg @@ -88,6 +94,7 @@ The following alternative symbols are supported: Electric current is expressed in amperes (symbol: `A`). The following alternative symbols are supported: + * `Bi`: daA * `Gb`: cm·(A/m)*250.0/[one?] @@ -96,6 +103,7 @@ The following alternative symbols are supported: Temperature is expressed in kelvins (symbol: `K`). The following alternative symbols are supported: + * `Cel`: ℃ @@ -111,6 +119,7 @@ Luminous intensity is expressed in candelas (symbol: `cd`). Area is expressed in square metres (symbol: `m²`). The following alternative symbols are supported: + * `m2`: m² * `ar`: hm² * `syd`: ((c(cm*254.0))*12.0)*3.0² @@ -124,6 +133,7 @@ The following alternative symbols are supported: Volume is expressed in cubic metres (symbol: `㎥`). The following alternative symbols are supported: + * `st`: [㎥?] * `bf`: (c(cm*254.0)³)*144.0 * `cyd`: ((c(cm*254.0))*12.0)*3.0³ @@ -139,6 +149,7 @@ The following alternative symbols are supported: Energy is expressed in joules (symbol: `J`). The following alternative symbols are supported: + * `cal_IT`: (J*41868.0)/10000.0 * `eV`: J*1.602176487E-19 * `cal_m`: (J*419002.0)/100000.0 @@ -150,6 +161,7 @@ The following alternative symbols are supported: Angle is expressed in radians (symbol: `rad`). The following alternative symbols are supported: + * `circ`: [one?]·rad*2.0 * `gon`: ([one?]·rad/180.0)*0.9 * `deg`: [one?]·rad/180.0 @@ -159,40 +171,40 @@ The following alternative symbols are supported: ### Others -- `1/m`, derived units: `Ky`: c(1/m) -- `kg/(m·s)`, derived units: `P`: g/(s·cm) -- `bit/s`, derived units: `Bd`: bit/s -- `bit`, derived units: `By`: bit*8.0 -- `Sv` -- `N` -- `Ω`, derived units: `Ohm`: Ω -- `T`, derived units: `G`: T/10000.0 -- `sr`, derived units: `sph`: [one?]·sr*4.0 -- `F` -- `C/kg`, derived units: `R`: (C/kg)*2.58E-4 -- `cd/m²`, derived units: `sb`: cd/cm², `Lmb`: cd/([one?]·cm²) -- `Pa`, derived units: `bar`: Pa*100000.0, `atm`: Pa*101325.0 -- `kg/(m·s²)`, derived units: `att`: k(g·(m/s²)*9.80665)/cm² -- `m²/s`, derived units: `St`: cm²/s -- `A/m`, derived units: `Oe`: (A/m)*250.0/[one?] -- `kg·m²/s²`, derived units: `erg`: cm²·g/s² -- `kg/m³`, derived units: `g%`: g/dl -- `mho` -- `V` -- `lx`, derived units: `ph`: lx/10000.0 -- `m/s²`, derived units: `Gal`: cm/s², `m/s2`: m/s² -- `m/s`, derived units: `kn`: m*1852.0/h -- `m·kg/s²`, derived units: `gf`: g·(m/s²)*9.80665, `lbf`: lb·(m/s²)*9.80665, `dyn`: cm·g/s² -- `m²/s²`, derived units: `RAD`: cm²·g/(s²·hg), `REM`: cm²·g/(s²·hg) -- `C` -- `Gy` -- `Hz` -- `H` -- `lm` -- `W` -- `Wb`, derived units: `Mx`: Wb/1.0E8 -- `Bq`, derived units: `Ci`: Bq*3.7E10 -- `S` +* `1/m`, derived units: `Ky`: c(1/m) +* `kg/(m·s)`, derived units: `P`: g/(s·cm) +* `bit/s`, derived units: `Bd`: bit/s +* `bit`, derived units: `By`: bit*8.0 +* `Sv` +* `N` +* `Ω`, derived units: `Ohm`: Ω +* `T`, derived units: `G`: T/10000.0 +* `sr`, derived units: `sph`: [one?]·sr*4.0 +* `F` +* `C/kg`, derived units: `R`: (C/kg)*2.58E-4 +* `cd/m²`, derived units: `sb`: cd/cm², `Lmb`: cd/([one?]·cm²) +* `Pa`, derived units: `bar`: Pa*100000.0, `atm`: Pa*101325.0 +* `kg/(m·s²)`, derived units: `att`: k(g·(m/s²)*9.80665)/cm² +* `m²/s`, derived units: `St`: cm²/s +* `A/m`, derived units: `Oe`: (A/m)*250.0/[one?] +* `kg·m²/s²`, derived units: `erg`: cm²·g/s² +* `kg/m³`, derived units: `g%`: g/dl +* `mho` +* `V` +* `lx`, derived units: `ph`: lx/10000.0 +* `m/s²`, derived units: `Gal`: cm/s², `m/s2`: m/s² +* `m/s`, derived units: `kn`: m*1852.0/h +* `m·kg/s²`, derived units: `gf`: g·(m/s²)*9.80665, `lbf`: lb·(m/s²)*9.80665, `dyn`: cm·g/s² +* `m²/s²`, derived units: `RAD`: cm²·g/(s²·hg), `REM`: cm²·g/(s²·hg) +* `C` +* `Gy` +* `Hz` +* `H` +* `lm` +* `W` +* `Wb`, derived units: `Mx`: Wb/1.0E8 +* `Bq`, derived units: `Ci`: Bq*3.7E10 +* `S` ## Examples @@ -339,9 +351,9 @@ The following alternative symbols are supported: Target unit. Can be left empty to convert to the respective SI base units. -- ID: `targetUnit` -- Datatype: `string` -- Default Value: `None` +* ID: `targetUnit` +* Datatype: `string` +* Default Value: `None` @@ -349,9 +361,9 @@ Target unit. Can be left empty to convert to the respective SI base units. The IETF BCP 47 language tag, e.g., 'en'. -- ID: `numberFormat` -- Datatype: `string` -- Default Value: `en` +* ID: `numberFormat` +* Datatype: `string` +* Default Value: `en` @@ -359,4 +371,4 @@ The IETF BCP 47 language tag, e.g., 'en'. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Numeric/aggregateNumbers.md b/docs/build/reference/transformer/Numeric/aggregateNumbers.md index 68ee3364..6e2a6709 100644 --- a/docs/build/reference/transformer/Numeric/aggregateNumbers.md +++ b/docs/build/reference/transformer/Numeric/aggregateNumbers.md @@ -2,10 +2,12 @@ title: "Aggregate numbers" description: "Applies one of the aggregation operators (`+`, `*`, `min`, `max` or `average`) to the sequence of input values." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Aggregate numbers + @@ -234,9 +236,9 @@ numbers, they will be ignored. The aggregation operation to be applied to all values. One of `+`, `*`, `min`, `max`, `average`. -- ID: `operator` -- Datatype: `string` -- Default Value: `None` +* ID: `operator` +* Datatype: `string` +* Default Value: `None` @@ -244,4 +246,4 @@ The aggregation operation to be applied to all values. One of `+`, `*`, `min`, ` ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Numeric/cmem-plugin-number-conversion.md b/docs/build/reference/transformer/Numeric/cmem-plugin-number-conversion.md index b12f8842..26181070 100644 --- a/docs/build/reference/transformer/Numeric/cmem-plugin-number-conversion.md +++ b/docs/build/reference/transformer/Numeric/cmem-plugin-number-conversion.md @@ -2,11 +2,13 @@ title: "Convert Number Base" description: "Convert numbers between different number bases (binary, octal, decimal, hexadecimal)." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator - PythonPlugin --- + # Convert Number Base + !!! note inline end "Python Plugin" @@ -45,4 +47,4 @@ Source Number Base ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Numeric/compareNumbers.md b/docs/build/reference/transformer/Numeric/compareNumbers.md index 01583004..411df8c7 100644 --- a/docs/build/reference/transformer/Numeric/compareNumbers.md +++ b/docs/build/reference/transformer/Numeric/compareNumbers.md @@ -2,10 +2,12 @@ title: "Compare numbers" description: "Compares the numbers of two sets. Returns 1 if the comparison yields true and 0 otherwise. If there are multiple numbers in both sets, the comparator must be true for all numbers. For instance, {1,2} < {2,3} yields 0 as not all numbers in the first set are smaller than in the second." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Compare numbers + @@ -32,4 +34,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Numeric/extractPhysicalQuantity.md b/docs/build/reference/transformer/Numeric/extractPhysicalQuantity.md index a35b4977..e83d4289 100644 --- a/docs/build/reference/transformer/Numeric/extractPhysicalQuantity.md +++ b/docs/build/reference/transformer/Numeric/extractPhysicalQuantity.md @@ -2,10 +2,12 @@ title: "Extract physical quantity" description: "Extracts physical quantities, such as length or weight values. Values are expected to be formatted as `{Number}{UnitPrefix}{Symbol}` and are converted to the base unit." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Extract physical quantity + @@ -62,4 +64,4 @@ If there are multiple matches, retrieve the value with the given index (zero-bas ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Numeric/formatNumber.md b/docs/build/reference/transformer/Numeric/formatNumber.md index 85e65115..8ed9d91e 100644 --- a/docs/build/reference/transformer/Numeric/formatNumber.md +++ b/docs/build/reference/transformer/Numeric/formatNumber.md @@ -2,10 +2,12 @@ title: "Format number" description: "Formats a number according to a user-defined pattern. The pattern syntax is documented at: https://docs.oracle.com/javase/8/docs/api/java/text/DecimalFormat.html" icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Format number + @@ -13,7 +15,7 @@ tags: Formats a number according to a user-defined pattern. The pattern syntax is documented at: - https://docs.oracle.com/javase/8/docs/api/java/text/DecimalFormat.html + ## Examples @@ -125,9 +127,9 @@ tags: No description -- ID: `pattern` -- Datatype: `string` -- Default Value: `None` +* ID: `pattern` +* Datatype: `string` +* Default Value: `None` @@ -135,9 +137,9 @@ No description No description -- ID: `locale` -- Datatype: `string` -- Default Value: `en` +* ID: `locale` +* Datatype: `string` +* Default Value: `en` @@ -145,4 +147,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Numeric/log.md b/docs/build/reference/transformer/Numeric/log.md index 342684e4..308b9ef1 100644 --- a/docs/build/reference/transformer/Numeric/log.md +++ b/docs/build/reference/transformer/Numeric/log.md @@ -2,10 +2,12 @@ title: "Logarithm" description: "Transforms all numbers by applying the logarithm function. Non-numeric values are left unchanged." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Logarithm + @@ -29,4 +31,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Numeric/numOperation.md b/docs/build/reference/transformer/Numeric/numOperation.md index dac6664e..81499e24 100644 --- a/docs/build/reference/transformer/Numeric/numOperation.md +++ b/docs/build/reference/transformer/Numeric/numOperation.md @@ -2,10 +2,12 @@ title: "Numeric operation" description: "Applies one of the four basic arithmetic operators to the sequence of input values." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Numeric operation + @@ -146,9 +148,9 @@ Division is configured with the `/` operator. The symbol `÷` is not supported. The operator to be applied to all values. One of `+`, `-`, `*`, `/` -- ID: `operator` -- Datatype: `string` -- Default Value: `None` +* ID: `operator` +* Datatype: `string` +* Default Value: `None` @@ -156,4 +158,4 @@ The operator to be applied to all values. One of `+`, `-`, `*`, `/` ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Numeric/numReduce.md b/docs/build/reference/transformer/Numeric/numReduce.md index 85114bb7..e4c34c01 100644 --- a/docs/build/reference/transformer/Numeric/numReduce.md +++ b/docs/build/reference/transformer/Numeric/numReduce.md @@ -2,10 +2,12 @@ title: "Numeric reduce" description: "Strips all non-numeric characters from a string." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Numeric reduce + @@ -48,9 +50,9 @@ Strips all non-numeric characters from a string. No description -- ID: `keepPunctuation` -- Datatype: `boolean` -- Default Value: `true` +* ID: `keepPunctuation` +* Datatype: `boolean` +* Default Value: `true` @@ -58,4 +60,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Parser/DateTypeParser.md b/docs/build/reference/transformer/Parser/DateTypeParser.md index d702de51..fc531d24 100644 --- a/docs/build/reference/transformer/Parser/DateTypeParser.md +++ b/docs/build/reference/transformer/Parser/DateTypeParser.md @@ -2,10 +2,12 @@ title: "Parse date" description: "Parses and normalizes dates in different formats." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Parse date + @@ -256,9 +258,9 @@ Parses and normalizes dates in different formats. The input date/time format used for parsing the date/time string. -- ID: `inputDateFormatId` -- Datatype: `option[enumeration]` -- Default Value: `w3c Date` +* ID: `inputDateFormatId` +* Datatype: `option[enumeration]` +* Default Value: `w3c Date` @@ -266,9 +268,9 @@ The input date/time format used for parsing the date/time string. An input format string that should be used instead of the selected input format. Java DateFormat string. -- ID: `alternativeInputFormat` -- Datatype: `string` -- Default Value: `None` +* ID: `alternativeInputFormat` +* Datatype: `string` +* Default Value: `None` @@ -276,9 +278,9 @@ An input format string that should be used instead of the selected input format. Optional locale for the (alternative) input format. If not set the system's locale will be used or the locale of the input format, if set. -- ID: `inputLocale` -- Datatype: `option[locale]` -- Default Value: `None` +* ID: `inputLocale` +* Datatype: `option[locale]` +* Default Value: `None` @@ -286,9 +288,9 @@ Optional locale for the (alternative) input format. If not set the system's loca The output date/time format used for parsing the date/time string. -- ID: `outputDateFormatId` -- Datatype: `option[enumeration]` -- Default Value: `w3c Date` +* ID: `outputDateFormatId` +* Datatype: `option[enumeration]` +* Default Value: `w3c Date` @@ -296,9 +298,9 @@ The output date/time format used for parsing the date/time string. An output format string that should be used instead of the selected output format. Java DateFormat string. -- ID: `alternativeOutputFormat` -- Datatype: `string` -- Default Value: `None` +* ID: `alternativeOutputFormat` +* Datatype: `string` +* Default Value: `None` @@ -306,9 +308,9 @@ An output format string that should be used instead of the selected output forma Optional locale for the (alternative) output format. If not set the system's locale will be used or the locale of the output format, if set. -- ID: `outputLocale` -- Datatype: `option[locale]` -- Default Value: `None` +* ID: `outputLocale` +* Datatype: `option[locale]` +* Default Value: `None` @@ -316,4 +318,4 @@ Optional locale for the (alternative) output format. If not set the system's loc ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Parser/FloatTypeParser.md b/docs/build/reference/transformer/Parser/FloatTypeParser.md index da57a859..0735c92c 100644 --- a/docs/build/reference/transformer/Parser/FloatTypeParser.md +++ b/docs/build/reference/transformer/Parser/FloatTypeParser.md @@ -2,10 +2,12 @@ title: "Parse float" description: "Parses and normalizes float values." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Parse float + @@ -49,4 +51,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Parser/GeoCoordinateParser.md b/docs/build/reference/transformer/Parser/GeoCoordinateParser.md index ff7e8a4f..697b7b4f 100644 --- a/docs/build/reference/transformer/Parser/GeoCoordinateParser.md +++ b/docs/build/reference/transformer/Parser/GeoCoordinateParser.md @@ -2,10 +2,12 @@ title: "Parse geo coordinate" description: "Parses and normalizes geo coordinates." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Parse geo coordinate + @@ -24,15 +26,15 @@ The origin of this coordinate system is known as the [Null Island](https://en.wi Geographic coordinates are dimensionless numbers for measuring and expressing **angles**. By convention, these angles are expressed in **degrees**, not in radians. This means, for the range of values for **latitude** and **longitude**, the following: -- **Latitude (φ):** range **–90° ≤ φ ≤ +90°** - - **–90°** → South Pole - - **0°** → Equator - - **+90°** → North Pole +* **Latitude (φ):** range **–90° ≤ φ ≤ +90°** + * **–90°** → South Pole + * **0°** → Equator + * **+90°** → North Pole -- **Longitude (λ):** range **–180° ≤ λ ≤ +180°** - - **0°** → Prime Meridian (Greenwich) - - **+180°** → directly east of Greenwich (International Date Line eastward) - - **–180°** → directly west of Greenwich (International Date Line westward) +* **Longitude (λ):** range **–180° ≤ λ ≤ +180°** + * **0°** → Prime Meridian (Greenwich) + * **+180°** → directly east of Greenwich (International Date Line eastward) + * **–180°** → directly west of Greenwich (International Date Line westward) The meridian at 180° is known as the **antimeridian** or, simply, the **180th meridian**. @@ -42,18 +44,20 @@ The meridian at 180° is known as the **antimeridian** or, simply, the **180th m The input is expressed in degrees-minutes-seconds (DMS). Each portion is accompanied by the corresponding symbol: -- **Degrees (°)** -- **Minutes (′)** -- **Seconds (″)** +* **Degrees (°)** +* **Minutes (′)** +* **Seconds (″)** + #### Example An example of the normalization of the geographic coordinates `"51°20.519' N,12°22.443' E"` is `(51.3419833, 12.37405)`. This stems from the following calculation: - Latitude = 51 + 1⁄60 · 20.519 = 51.3419833° N, - Longitude = 12 + 1⁄60 · 22.443 = 12.37405° E. + Latitude = 51 + 1⁄60 · 20.519 = 51.3419833° N, + Longitude = 12 + 1⁄60 · 22.443 = 12.37405° E. A bit more detailed: -1. First, the input is _parsed_: The **latitude** is `51°20.519' N`, the **longitude** is `51°20.519' N`. As stated, **north** (`N`) and **east** (`E`) are *positive* by convention. Correspondingly, **south** (`S`) and **west** (`w`) are *negative*. + +1. First, the input is _parsed_: The **latitude** is `51°20.519' N`, the **longitude** is `51°20.519' N`. As stated, **north** (`N`) and **east** (`E`) are _positive_ by convention. Correspondingly, **south** (`S`) and **west** (`w`) are _negative_. 2. Next, _each_ of these **components** is split into their **portions**: * `51°20.519' N` corresponds to 51 degrees and 20.519 minutes. * Notice that the portion **`20.519` minutes** contains, itself, a _decimal_ number, instead of the degrees-**minutes-seconds** format. This is because decimal degrees and minutes are permitted as a form of mixed format by our plugin. @@ -61,7 +65,7 @@ A bit more detailed: * `12°22.443' E` corresponds to 12 degrees, 22.443 minutes. * Once more, the portion `22.443 minutes` is a mixed format, where instead of minutes and seconds, we write the minutes in decimal format to include the seconds. 3. Finally, the DMS format of the input is **normalized** according to the following conversion formula: - decimal degrees = *degrees* + 1⁄60 · minutes + 1⁄3600 · seconds. + decimal degrees = _degrees_ + 1⁄60 · minutes + 1⁄3600 · seconds. ## Examples @@ -120,4 +124,4 @@ A bit more detailed: ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Parser/GeoLocationParser.md b/docs/build/reference/transformer/Parser/GeoLocationParser.md index 6913a7e2..53ae0600 100644 --- a/docs/build/reference/transformer/Parser/GeoLocationParser.md +++ b/docs/build/reference/transformer/Parser/GeoLocationParser.md @@ -2,10 +2,12 @@ title: "Parse geo location" description: "Parses and normalizes geo locations like continents, countries, states and cities." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Parse geo location + @@ -13,6 +15,7 @@ tags: ## What does this plugin do? This plugin **parses and normalizes geolocations**. + ### What is Geolocation? The term "geolocation" means two things. Usually, it is used primarily as a noun referring to the **process** or technique of _determining the physical location_ of a person or an item, e.g. by means of GPS or IP addresses. On the other hand – and this is the meaning of the word that _we_'ll use – it is also used to describe the _actual location_ that is identified. @@ -51,6 +54,7 @@ The `continent` parameter refers to a _continent code_. Although these contintne | North America | NA | | | Oceania | OC | We don't use the alternative `UA` | | South America | SA | | + ##### Country and Numeric Country Both `country` and `numericCountry` refer to the [ISO 3166-1 alpha-3](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-3) country code, such as `AND` for Andorra, as can be found in the list of [officially assigned ISO 3166-1 alpha-3 codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-3#Officially_assigned_code_elements). @@ -280,9 +284,9 @@ As a last level of geospatial precision, we have the `city`. Currently, this doe What type of location should be parsed. -- ID: `parseTypeId` -- Datatype: `enumeration` -- Default Value: `None` +* ID: `parseTypeId` +* Datatype: `enumeration` +* Default Value: `None` @@ -290,9 +294,9 @@ What type of location should be parsed. Set to true if the full state name should be output instead of the 2-letter code. -- ID: `fullStateName` -- Datatype: `boolean` -- Default Value: `true` +* ID: `fullStateName` +* Datatype: `boolean` +* Default Value: `true` @@ -300,4 +304,4 @@ Set to true if the full state name should be output instead of the 2-letter code ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Parser/IntegerParser.md b/docs/build/reference/transformer/Parser/IntegerParser.md index 63f6d890..50f390b7 100644 --- a/docs/build/reference/transformer/Parser/IntegerParser.md +++ b/docs/build/reference/transformer/Parser/IntegerParser.md @@ -2,10 +2,12 @@ title: "Parse integer" description: "Parses integer values." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Parse integer + @@ -76,9 +78,9 @@ Parses integer values. Use comma or point (default) as a decimal separator. -- ID: `commaAsDecimalPoint` -- Datatype: `boolean` -- Default Value: `false` +* ID: `commaAsDecimalPoint` +* Datatype: `boolean` +* Default Value: `false` @@ -86,9 +88,9 @@ Use comma or point (default) as a decimal separator. Presence of a thousands separator (default: absence), compatible with the chosen decimal separator. -- ID: `thousandSeparator` -- Datatype: `boolean` -- Default Value: `false` +* ID: `thousandSeparator` +* Datatype: `boolean` +* Default Value: `false` @@ -96,4 +98,4 @@ Presence of a thousands separator (default: absence), compatible with the chosen ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Parser/IsinParser.md b/docs/build/reference/transformer/Parser/IsinParser.md index 8ce1a4e4..2724b88b 100644 --- a/docs/build/reference/transformer/Parser/IsinParser.md +++ b/docs/build/reference/transformer/Parser/IsinParser.md @@ -2,10 +2,12 @@ title: "Parse ISIN" description: "Parses International Securities Identification Numbers (ISIN) values and fails if the String is no valid ISIN." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Parse ISIN + @@ -19,4 +21,4 @@ Parses International Securities Identification Numbers (ISIN) values and fails i ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Parser/SkosTypeParser.md b/docs/build/reference/transformer/Parser/SkosTypeParser.md index d8f7b29b..e42678b2 100644 --- a/docs/build/reference/transformer/Parser/SkosTypeParser.md +++ b/docs/build/reference/transformer/Parser/SkosTypeParser.md @@ -2,10 +2,12 @@ title: "Parse SKOS term" description: "Parses values from a SKOS ontology." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Parse SKOS term + @@ -29,4 +31,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Parser/StringParser.md b/docs/build/reference/transformer/Parser/StringParser.md index 19a76ee4..f8eeb7b2 100644 --- a/docs/build/reference/transformer/Parser/StringParser.md +++ b/docs/build/reference/transformer/Parser/StringParser.md @@ -2,10 +2,12 @@ title: "Parse string" description: "Parses string values. This is basically an identity function." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Parse string + @@ -42,4 +44,4 @@ Parses string values. This is basically an identity function. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Replace/excelMap.md b/docs/build/reference/transformer/Replace/excelMap.md index 9c9ae084..66ce4f2b 100644 --- a/docs/build/reference/transformer/Replace/excelMap.md +++ b/docs/build/reference/transformer/Replace/excelMap.md @@ -1,21 +1,25 @@ --- title: "Excel map" -description: "Replaces values based on a map of values read from a file in Open XML format (XLSX). The XLSX file may contain several sheets of the form: ``` mapFrom,mapTo , ``` and more. An empty string can be created in Excel and alternatives by inserting `=''` in the input line of a cell. If there are multiple values for a single key, all values will be returned for the given key. Note that the mapping table will be cached in memory. If the Excel file is updated (even while transforming), the map will be reloaded within seconds." +description: "Replaces values based on a map of values read from a file in Open XML format (XLSX). The XLSX file may contain several sheets of the form: ```mapFrom,mapTo ,``` and more. An empty string can be created in Excel and alternatives by inserting `=''` in the input line of a cell. If there are multiple values for a single key, all values will be returned for the given key. Note that the mapping table will be cached in memory. If the Excel file is updated (even while transforming), the map will be reloaded within seconds." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Excel map + Replaces values based on a map of values read from a file in Open XML format (XLSX). The XLSX file may contain several sheets of the form: -``` + +```text mapFrom,mapTo , ``` + and more. An empty string can be created in Excel and alternatives by inserting `=""` in the input line of a cell. @@ -23,7 +27,7 @@ An empty string can be created in Excel and alternatives by inserting `=""` in t If there are multiple values for a single key, all values will be returned for the given key. Note that the mapping table will be cached in memory. If the Excel file is updated (even while transforming), the map will be reloaded within seconds. - + ## Parameter @@ -82,4 +86,4 @@ Determines how values that cannot be found in the mapping table are treated. Onl ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Replace/map.md b/docs/build/reference/transformer/Replace/map.md index 7ee0d0a5..902e639b 100644 --- a/docs/build/reference/transformer/Replace/map.md +++ b/docs/build/reference/transformer/Replace/map.md @@ -2,10 +2,12 @@ title: "Map" description: "Replaces values based on a map of values." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Map + @@ -50,9 +52,9 @@ Replaces values based on a map of values. A map of values -- ID: `map` -- Datatype: `stringmap` -- Default Value: `None` +* ID: `map` +* Datatype: `stringmap` +* Default Value: `None` @@ -60,9 +62,9 @@ A map of values Default if the map defines no value -- ID: `default` -- Datatype: `string` -- Default Value: `None` +* ID: `default` +* Datatype: `string` +* Default Value: `None` @@ -70,4 +72,4 @@ Default if the map defines no value ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Replace/mapWithDefaultInput.md b/docs/build/reference/transformer/Replace/mapWithDefaultInput.md index c8c818dd..052841e4 100644 --- a/docs/build/reference/transformer/Replace/mapWithDefaultInput.md +++ b/docs/build/reference/transformer/Replace/mapWithDefaultInput.md @@ -2,10 +2,12 @@ title: "Map with default" description: "Maps input values from the first input using a predefined map, with fallback to default values provided by the second input." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Map with default + @@ -15,9 +17,10 @@ This transformer requires a _map of values_, when created. This can be a map suc The transformer requires _two_ input value sequences, when called: the first sequence of values are the _values to map_, and the second is a sequence of _default values_. With these parameterization and applied value sequences, the transformer then works in the following way: -* The _map of values_ (specified when the transformer is _created_) is used for _obtaining_ values from the transformer. -* The _values to map_ (specified when the transformer is _called_) is used for _mapping_ values by the transformer. -* The _default values_ (specified when the transformer is _called_, as a mandatory second argument) is used as a backup sequence of values, in case the (first) value to map is not found within the map of values. It is simply a default. + +- The _map of values_ (specified when the transformer is _created_) is used for _obtaining_ values from the transformer. +- The _values to map_ (specified when the transformer is _called_) is used for _mapping_ values by the transformer. +- The _default values_ (specified when the transformer is _called_, as a mandatory second argument) is used as a backup sequence of values, in case the (first) value to map is not found within the map of values. It is simply a default. Normally, the sequence of _default values_ is expected to have the same size as the _values to map_ (i.e. the two sequences provided when _calling_ the transformer are supposed to be compatible). Additionally, in order to provide a certain amount of flexibility: Should that _not_ be the case, if there are _less_ default values than values to map, the _last_ default value is replicated to match the count. This fallback shouldn't be relied upon, since it may result in a somewhat confusing or unexpected behavior. @@ -38,4 +41,4 @@ A map of values ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Replace/regexReplace.md b/docs/build/reference/transformer/Replace/regexReplace.md index b40d99fc..06edcebf 100644 --- a/docs/build/reference/transformer/Replace/regexReplace.md +++ b/docs/build/reference/transformer/Replace/regexReplace.md @@ -2,10 +2,12 @@ title: "Regex replace" description: "Replace all occurrences of a regular expression in a string. If no replacement is given, the occurrences of the regular expression will be deleted." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Regex replace + @@ -122,9 +124,9 @@ A compilation of the available constructs for building regular expressions is av The regular expression to match -- ID: `regex` -- Datatype: `string` -- Default Value: `None` +* ID: `regex` +* Datatype: `string` +* Default Value: `None` @@ -132,9 +134,9 @@ The regular expression to match The replacement of each match -- ID: `replace` -- Datatype: `string` -- Default Value: `None` +* ID: `replace` +* Datatype: `string` +* Default Value: `None` @@ -142,4 +144,4 @@ The replacement of each match ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Replace/replace.md b/docs/build/reference/transformer/Replace/replace.md index 4dd13c6b..48ef58d2 100644 --- a/docs/build/reference/transformer/Replace/replace.md +++ b/docs/build/reference/transformer/Replace/replace.md @@ -2,10 +2,12 @@ title: "Replace" description: "Replace all occurrences of a string with another string." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Replace + @@ -20,7 +22,7 @@ Replace all occurrences of a string with another string. **Example 1:** * Parameters - * search: ` ` + * search: `` * replace: `` * Input values: @@ -50,9 +52,9 @@ Replace all occurrences of a string with another string. The string to search for -- ID: `search` -- Datatype: `string` -- Default Value: `None` +* ID: `search` +* Datatype: `string` +* Default Value: `None` @@ -60,9 +62,9 @@ The string to search for The replacement of each match -- ID: `replace` -- Datatype: `string` -- Default Value: `None` +* ID: `replace` +* Datatype: `string` +* Default Value: `None` @@ -70,4 +72,4 @@ The replacement of each match ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Selection/coalesce.md b/docs/build/reference/transformer/Selection/coalesce.md index 1f648a09..dfdf00c3 100644 --- a/docs/build/reference/transformer/Selection/coalesce.md +++ b/docs/build/reference/transformer/Selection/coalesce.md @@ -2,10 +2,12 @@ title: "Coalesce (first non-empty input)" description: "Forwards the first non-empty input, i.e. for which any value(s) exist. A single empty string is considered a value." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Coalesce (first non-empty input) + @@ -83,4 +85,4 @@ Forwards the first non-empty input, i.e. for which any value(s) exist. A single ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Selection/regexSelect.md b/docs/build/reference/transformer/Selection/regexSelect.md index 8b29125c..1758dffe 100644 --- a/docs/build/reference/transformer/Selection/regexSelect.md +++ b/docs/build/reference/transformer/Selection/regexSelect.md @@ -2,10 +2,12 @@ title: "Regex selection" description: "This transformer takes 3 inputs: one output value, multiple regex patterns, and a value to check against those patterns. It returns the output value at positions where regex patterns match the input value." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Regex selection + @@ -83,9 +85,9 @@ A compilation of the available constructs for building regular expressions is av No description -- ID: `oneOnly` -- Datatype: `boolean` -- Default Value: `false` +* ID: `oneOnly` +* Datatype: `boolean` +* Default Value: `false` @@ -93,4 +95,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Sequence/count.md b/docs/build/reference/transformer/Sequence/count.md index 944eb88d..936ae9f9 100644 --- a/docs/build/reference/transformer/Sequence/count.md +++ b/docs/build/reference/transformer/Sequence/count.md @@ -2,10 +2,12 @@ title: "Count values" description: "Counts the number of values." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Count values + @@ -42,4 +44,4 @@ Counts the number of values. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Sequence/getValueByIndex.md b/docs/build/reference/transformer/Sequence/getValueByIndex.md index 5e122f0f..45b73eb5 100644 --- a/docs/build/reference/transformer/Sequence/getValueByIndex.md +++ b/docs/build/reference/transformer/Sequence/getValueByIndex.md @@ -2,10 +2,12 @@ title: "Get value by index" description: "Returns the value found at the specified index. Fails or returns an empty result depending on failIfNoFound is set or not. Please be aware that this will work only if the data source supports some kind of ordering like XML or JSON. This is probably not a good idea to do with RDF models. If emptyStringToEmptyResult is true then instead of a result with an empty String, an empty result is returned." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Get value by index + @@ -15,7 +17,7 @@ Returns the value found at the specified index. Fails or returns an empty result is probably not a good idea to do with RDF models. If emptyStringToEmptyResult is true then instead of a result with an empty String, an empty result is returned. - + ## Parameter @@ -54,4 +56,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Sequence/sort.md b/docs/build/reference/transformer/Sequence/sort.md index ed352f5d..8042813c 100644 --- a/docs/build/reference/transformer/Sequence/sort.md +++ b/docs/build/reference/transformer/Sequence/sort.md @@ -2,10 +2,12 @@ title: "Sort" description: "Sorts values lexicographically." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Sort + @@ -51,4 +53,4 @@ Sorts values lexicographically. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Sequence/toSequenceIndex.md b/docs/build/reference/transformer/Sequence/toSequenceIndex.md index e83a72a9..afbda059 100644 --- a/docs/build/reference/transformer/Sequence/toSequenceIndex.md +++ b/docs/build/reference/transformer/Sequence/toSequenceIndex.md @@ -2,10 +2,12 @@ title: "Sequence values to indexes" description: "Transforms the sequence of values to their respective indexes in the sequence. If there is more than one input, the values are numbered from the first input on and continued for the next inputs. Applied against an RDF source the order might not be deterministic." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Sequence values to indexes + @@ -33,4 +35,4 @@ Transforms the sequence of values to their respective indexes in the sequence. I ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Substring/stripPostfix.md b/docs/build/reference/transformer/Substring/stripPostfix.md index fad94397..13f861c3 100644 --- a/docs/build/reference/transformer/Substring/stripPostfix.md +++ b/docs/build/reference/transformer/Substring/stripPostfix.md @@ -2,10 +2,12 @@ title: "Strip postfix" description: "Strips a postfix of a string." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Strip postfix + @@ -48,9 +50,9 @@ Strips a postfix of a string. No description -- ID: `postfix` -- Datatype: `string` -- Default Value: `None` +* ID: `postfix` +* Datatype: `string` +* Default Value: `None` @@ -58,4 +60,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Substring/stripPrefix.md b/docs/build/reference/transformer/Substring/stripPrefix.md index 14a030d3..733709ee 100644 --- a/docs/build/reference/transformer/Substring/stripPrefix.md +++ b/docs/build/reference/transformer/Substring/stripPrefix.md @@ -2,10 +2,12 @@ title: "Strip prefix" description: "Strips a prefix of a string." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Strip prefix + @@ -48,9 +50,9 @@ Strips a prefix of a string. No description -- ID: `prefix` -- Datatype: `string` -- Default Value: `None` +* ID: `prefix` +* Datatype: `string` +* Default Value: `None` @@ -58,4 +60,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Substring/stripUriPrefix.md b/docs/build/reference/transformer/Substring/stripUriPrefix.md index 45327169..f5771ac0 100644 --- a/docs/build/reference/transformer/Substring/stripUriPrefix.md +++ b/docs/build/reference/transformer/Substring/stripUriPrefix.md @@ -2,10 +2,12 @@ title: "Strip URI prefix" description: "Strips the URI prefix and decodes the remainder based on UTF-8 URL decoding (using java.net.URLDecoder). Leaves values unchanged which are not a valid URI." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Strip URI prefix + @@ -81,9 +83,9 @@ Strips the URI prefix and decodes the remainder based on UTF-8 URL decoding (usi If true, underscores will be decoded to spaces. -- ID: `decodeUnderscoresToSpaces` -- Datatype: `boolean` -- Default Value: `true` +* ID: `decodeUnderscoresToSpaces` +* Datatype: `boolean` +* Default Value: `true` @@ -91,4 +93,4 @@ If true, underscores will be decoded to spaces. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Substring/substring.md b/docs/build/reference/transformer/Substring/substring.md index 6f0deaa1..54c80b02 100644 --- a/docs/build/reference/transformer/Substring/substring.md +++ b/docs/build/reference/transformer/Substring/substring.md @@ -2,10 +2,12 @@ title: "Substring" description: "Returns a substring between 'beginIndex' (inclusive) and 'endIndex' (exclusive). If 'endIndex' is 0 (default), it is ignored and the entire remaining string starting with 'beginIndex' is returned. If 'endIndex' is negative, -endIndex characters are removed from the end." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Substring + @@ -131,9 +133,9 @@ Returns a substring between 'beginIndex' (inclusive) and 'endIndex' (exclusive). The beginning index, inclusive. -- ID: `beginIndex` -- Datatype: `int` -- Default Value: `0` +* ID: `beginIndex` +* Datatype: `int` +* Default Value: `0` @@ -141,9 +143,9 @@ The beginning index, inclusive. The end index, exclusive. Ignored if set to 0, i.e., the entire remaining string starting with 'beginIndex' is returned. If negative, -endIndex characters are removed from the end. -- ID: `endIndex` -- Datatype: `int` -- Default Value: `0` +* ID: `endIndex` +* Datatype: `int` +* Default Value: `0` @@ -151,9 +153,9 @@ The end index, exclusive. Ignored if set to 0, i.e., the entire remaining string If true, only strings will be accepted that are within the start and end indices, throwing a validating error if an index is out of range. -- ID: `stringMustBeInRange` -- Datatype: `boolean` -- Default Value: `true` +* ID: `stringMustBeInRange` +* Datatype: `boolean` +* Default Value: `true` @@ -161,4 +163,4 @@ If true, only strings will be accepted that are within the start and end indices ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Substring/untilCharacter.md b/docs/build/reference/transformer/Substring/untilCharacter.md index 276bfd74..597df66f 100644 --- a/docs/build/reference/transformer/Substring/untilCharacter.md +++ b/docs/build/reference/transformer/Substring/untilCharacter.md @@ -2,10 +2,12 @@ title: "Until character" description: "Extracts the substring until the character given." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Until character + @@ -48,9 +50,9 @@ Extracts the substring until the character given. No description -- ID: `untilCharacter` -- Datatype: `char` -- Default Value: `None` +* ID: `untilCharacter` +* Datatype: `char` +* Default Value: `None` @@ -58,4 +60,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Template/TemplateTransformer.md b/docs/build/reference/transformer/Template/TemplateTransformer.md index dd6111b6..0182a485 100644 --- a/docs/build/reference/transformer/Template/TemplateTransformer.md +++ b/docs/build/reference/transformer/Template/TemplateTransformer.md @@ -2,10 +2,12 @@ title: "Evaluate template" description: "Evaluates a template. Input values can be addressed using the variables 'input1', 'input2', etc. Global variables are available in the 'global' scope, e.g., 'global.myVar'." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Evaluate template + @@ -20,10 +22,11 @@ Evaluates a template. Input values can be addressed using the variables 'input1' **Example 1:** * Parameters - * template: - ``` + * template: + ```text + Hello {{input1}} {{input2}}, - + How are you today? ``` @@ -31,10 +34,10 @@ Evaluates a template. Input values can be addressed using the variables 'input1' 1. `[John]` 2. `[Doe]` -* Returns: - ``` +* Returns: + ```text [Hello John Doe, - + How are you today?] ``` @@ -105,9 +108,9 @@ Evaluates a template. Input values can be addressed using the variables 'input1' The template -- ID: `template` -- Datatype: `template` -- Default Value: `None` +* ID: `template` +* Datatype: `template` +* Default Value: `None` @@ -115,9 +118,9 @@ The template The template language. Currently, Jinja is supported. -- ID: `language` -- Datatype: `string` -- Default Value: `jinja` +* ID: `language` +* Datatype: `string` +* Default Value: `jinja` @@ -125,4 +128,4 @@ The template language. Currently, Jinja is supported. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Tokenization/camelcasetokenizer.md b/docs/build/reference/transformer/Tokenization/camelcasetokenizer.md index bfd67cf2..d5da65d2 100644 --- a/docs/build/reference/transformer/Tokenization/camelcasetokenizer.md +++ b/docs/build/reference/transformer/Tokenization/camelcasetokenizer.md @@ -2,10 +2,12 @@ title: "Camel case tokenizer" description: "Tokenizes a camel case string. That is it splits strings between a lower case character and an upper case character." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Camel case tokenizer + @@ -42,4 +44,4 @@ Tokenizes a camel case string. That is it splits strings between a lower case ch ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Tokenization/tokenize.md b/docs/build/reference/transformer/Tokenization/tokenize.md index 61cad6aa..51a99d68 100644 --- a/docs/build/reference/transformer/Tokenization/tokenize.md +++ b/docs/build/reference/transformer/Tokenization/tokenize.md @@ -2,10 +2,12 @@ title: "Tokenize" description: "Tokenizes all input values." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Tokenize + @@ -45,9 +47,9 @@ Tokenizes all input values. The regular expression used to split values. -- ID: `regex` -- Datatype: `string` -- Default Value: `\s` +* ID: `regex` +* Datatype: `string` +* Default Value: `\s` @@ -55,4 +57,4 @@ The regular expression used to split values. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Uncategorized/cmem-plugin-jq-transform.md b/docs/build/reference/transformer/Uncategorized/cmem-plugin-jq-transform.md index 39543852..e89b3514 100644 --- a/docs/build/reference/transformer/Uncategorized/cmem-plugin-jq-transform.md +++ b/docs/build/reference/transformer/Uncategorized/cmem-plugin-jq-transform.md @@ -2,11 +2,13 @@ title: "jq" description: "Process a JSON path with a jq filter / program." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator - PythonPlugin --- + # jq + !!! note inline end "Python Plugin" @@ -22,7 +24,7 @@ tags: In order to test jq expressions, you can use [play.jqlang.org](https://play.jqlang.org/). -## Basic concepts: +## Basic concepts - Filters separated by a comma will produce multiple independent outputs: `,` - Will ignores error if the type is unexpected: `?` @@ -101,4 +103,3 @@ The jq program to apply to the input JSON string. - Default Value: `true` - diff --git a/docs/build/reference/transformer/Uncategorized/cmem_plugin_currencies-transform.md b/docs/build/reference/transformer/Uncategorized/cmem_plugin_currencies-transform.md index 88297004..d8796082 100644 --- a/docs/build/reference/transformer/Uncategorized/cmem_plugin_currencies-transform.md +++ b/docs/build/reference/transformer/Uncategorized/cmem_plugin_currencies-transform.md @@ -2,11 +2,13 @@ title: "Convert currency values" description: "Converts currencies values with current and historical exchange rates" icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator - PythonPlugin --- + # Convert currency values + !!! note inline end "Python Plugin" @@ -98,4 +100,3 @@ Instead of plain values, output additional background information. - Default Value: `false` - diff --git a/docs/build/reference/transformer/Validation/validateDateAfter.md b/docs/build/reference/transformer/Validation/validateDateAfter.md index 71e8a995..d4cdf037 100644 --- a/docs/build/reference/transformer/Validation/validateDateAfter.md +++ b/docs/build/reference/transformer/Validation/validateDateAfter.md @@ -2,10 +2,12 @@ title: "Validate date after" description: "Validates if the first input date is after the second input date. Outputs the first input if the validation is successful." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Validate date after + @@ -72,9 +74,9 @@ Validates if the first input date is after the second input date. Outputs the fi Allow both dates to be equal. -- ID: `allowEqual` -- Datatype: `boolean` -- Default Value: `false` +* ID: `allowEqual` +* Datatype: `boolean` +* Default Value: `false` @@ -82,4 +84,4 @@ Allow both dates to be equal. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Validation/validateDateRange.md b/docs/build/reference/transformer/Validation/validateDateRange.md index 3230371f..8aa180dc 100644 --- a/docs/build/reference/transformer/Validation/validateDateRange.md +++ b/docs/build/reference/transformer/Validation/validateDateRange.md @@ -2,10 +2,12 @@ title: "Validate date range" description: "Validates if dates are within a specified range." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Validate date range + @@ -39,4 +41,4 @@ Latest allowed data in YYYY-MM-DD ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Validation/validateNumberOfValues.md b/docs/build/reference/transformer/Validation/validateNumberOfValues.md index eab40703..7d2f18bd 100644 --- a/docs/build/reference/transformer/Validation/validateNumberOfValues.md +++ b/docs/build/reference/transformer/Validation/validateNumberOfValues.md @@ -2,10 +2,12 @@ title: "Validate number of values" description: "Validates that the number of values lies in a specified range." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Validate number of values + @@ -51,9 +53,9 @@ Validates that the number of values lies in a specified range. Minimum allowed number of values -- ID: `min` -- Datatype: `int` -- Default Value: `0` +* ID: `min` +* Datatype: `int` +* Default Value: `0` @@ -61,9 +63,9 @@ Minimum allowed number of values Maximum allowed number of values -- ID: `max` -- Datatype: `int` -- Default Value: `1` +* ID: `max` +* Datatype: `int` +* Default Value: `1` @@ -71,4 +73,4 @@ Maximum allowed number of values ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Validation/validateNumericRange.md b/docs/build/reference/transformer/Validation/validateNumericRange.md index 6d21e672..45d2f8ea 100644 --- a/docs/build/reference/transformer/Validation/validateNumericRange.md +++ b/docs/build/reference/transformer/Validation/validateNumericRange.md @@ -2,10 +2,12 @@ title: "Validate numeric range" description: "Validates if a number is within a specified range." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Validate numeric range + @@ -39,4 +41,4 @@ Maximum allowed number ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Validation/validateRegex.md b/docs/build/reference/transformer/Validation/validateRegex.md index a2dc2c7f..ac9b7881 100644 --- a/docs/build/reference/transformer/Validation/validateRegex.md +++ b/docs/build/reference/transformer/Validation/validateRegex.md @@ -2,10 +2,12 @@ title: "Validate regex" description: "Validates if all values match a regular expression." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Validate regex + @@ -124,9 +126,9 @@ A compilation of the available constructs for building regular expressions is av regular expression -- ID: `regex` -- Datatype: `string` -- Default Value: `\w*` +* ID: `regex` +* Datatype: `string` +* Default Value: `\w*` @@ -134,4 +136,4 @@ regular expression ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/cmem-plugin-ulid.md b/docs/build/reference/transformer/Value/cmem-plugin-ulid.md index e6c12bc0..906c96f1 100644 --- a/docs/build/reference/transformer/Value/cmem-plugin-ulid.md +++ b/docs/build/reference/transformer/Value/cmem-plugin-ulid.md @@ -2,11 +2,13 @@ title: "ULID" description: "Generate ULID strings - Universally Unique Lexicographically Sortable Identifiers." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator - PythonPlugin --- + # ULID + !!! note inline end "Python Plugin" @@ -59,4 +61,4 @@ Generate 'urn:x-ulid:*' strings. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID1.md b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID1.md index db48a0ae..bd0af4e7 100644 --- a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID1.md +++ b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID1.md @@ -2,11 +2,13 @@ title: "UUID1" description: "Generate a UUIDv1 from a host ID, sequence number, and the current time" icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator - PythonPlugin --- + # UUID1 + !!! note inline end "Python Plugin" @@ -47,4 +49,4 @@ If clock sequence is given, it is used as the sequence number. Otherwise a rando ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID1ToUUID6.md b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID1ToUUID6.md index 54b91d67..dc11470a 100644 --- a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID1ToUUID6.md +++ b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID1ToUUID6.md @@ -2,11 +2,13 @@ title: "UUID1 to UUID6" description: "Generate UUIDv6 from a UUIDv1." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator - PythonPlugin --- + # UUID1 to UUID6 + !!! note inline end "Python Plugin" @@ -28,4 +30,4 @@ not involve legacy UUIDv1 SHOULD consider using UUIDv7 instead. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID3.md b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID3.md index 4de98375..1c7c0411 100644 --- a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID3.md +++ b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID3.md @@ -2,11 +2,13 @@ title: "UUID3" description: "Generate a UUIDv3" icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator - PythonPlugin --- + # UUID3 + !!! note inline end "Python Plugin" @@ -44,4 +46,4 @@ Applies only if none of the pre-defined namespaces is selected. If enabled, the ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID4.md b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID4.md index e612c272..bd3319d6 100644 --- a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID4.md +++ b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID4.md @@ -2,11 +2,13 @@ title: "UUID4" description: "Generate a random UUIDv4." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator - PythonPlugin --- + # UUID4 + !!! note inline end "Python Plugin" @@ -23,4 +25,4 @@ UUIDv4 specifies a random UUID. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID5.md b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID5.md index 8a3e96c0..2cbeb1b9 100644 --- a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID5.md +++ b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID5.md @@ -2,11 +2,13 @@ title: "UUID5" description: "Generate a UUIDv5" icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator - PythonPlugin --- + # UUID5 + !!! note inline end "Python Plugin" @@ -44,4 +46,4 @@ Applies only if none of the pre-defined namespaces is selected. If enabled, the ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID6.md b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID6.md index 8f500ee5..7559e4ef 100644 --- a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID6.md +++ b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID6.md @@ -2,11 +2,13 @@ title: "UUID6" description: "Generate a UUIDv6 from a host ID, sequence number, and the current time" icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator - PythonPlugin --- + # UUID6 + !!! note inline end "Python Plugin" @@ -51,4 +53,4 @@ If clock sequence is given, it is used as the sequence number. Otherwise a rando ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID7.md b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID7.md index 0ce20ef3..f2259223 100644 --- a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID7.md +++ b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID7.md @@ -2,11 +2,13 @@ title: "UUID7" description: "Generate a UUIDv7 from a random number, and the current time." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator - PythonPlugin --- + # UUID7 + !!! note inline end "Python Plugin" @@ -30,4 +32,4 @@ Implementations SHOULD utilize UUIDv7 over UUIDv1 and ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID8.md b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID8.md index d97f8ced..b681a103 100644 --- a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID8.md +++ b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUID8.md @@ -2,11 +2,13 @@ title: "UUID8" description: "Generate a UUIDv8 from a random number, and the current time." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator - PythonPlugin --- + # UUID8 + !!! note inline end "Python Plugin" @@ -27,4 +29,4 @@ excluded. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUIDConvert.md b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUIDConvert.md index 1d996c8e..7a0161d0 100644 --- a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUIDConvert.md +++ b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUIDConvert.md @@ -2,11 +2,13 @@ title: "UUID Convert" description: "Convert a UUID string representation" icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator - PythonPlugin --- + # UUID Convert + !!! note inline end "Python Plugin" @@ -48,4 +50,4 @@ Output string format ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUIDVersion.md b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUIDVersion.md index 4800c0d8..acd95087 100644 --- a/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUIDVersion.md +++ b/docs/build/reference/transformer/Value/cmem_plugin_uuid-plugin_uuid-UUIDVersion.md @@ -2,11 +2,13 @@ title: "UUID Version" description: "Outputs UUID version number of input" icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator - PythonPlugin --- + # UUID Version + !!! note inline end "Python Plugin" @@ -23,4 +25,4 @@ Input: UUID string, output: UUID version number of input. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/constant.md b/docs/build/reference/transformer/Value/constant.md index b0eb141a..42f41909 100644 --- a/docs/build/reference/transformer/Value/constant.md +++ b/docs/build/reference/transformer/Value/constant.md @@ -2,10 +2,12 @@ title: "Constant" description: "Generates a constant value." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Constant + @@ -19,10 +21,10 @@ Generates a constant value. --- **Always outputs the specified value:** -* Parameters - * value: `John` +- Parameters + - value: `John` -* Returns: `[John]` +- Returns: `[John]` @@ -43,4 +45,4 @@ The constant value to be generated ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/constantUri.md b/docs/build/reference/transformer/Value/constantUri.md index f64f74ef..4a1303de 100644 --- a/docs/build/reference/transformer/Value/constantUri.md +++ b/docs/build/reference/transformer/Value/constantUri.md @@ -2,10 +2,12 @@ title: "Constant URI" description: "Generates a constant URI." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Constant URI + @@ -29,4 +31,4 @@ The constant URI to be generated ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/datasetParameter.md b/docs/build/reference/transformer/Value/datasetParameter.md index 6f0ffc84..1c95a4c7 100644 --- a/docs/build/reference/transformer/Value/datasetParameter.md +++ b/docs/build/reference/transformer/Value/datasetParameter.md @@ -2,10 +2,12 @@ title: "Dataset parameter" description: "Reads a meta data parameter from a dataset in Corporate Memory. If authentication is enabled, workbench.superuser must be configured." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Dataset parameter + @@ -59,4 +61,4 @@ No description ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/defaultValue.md b/docs/build/reference/transformer/Value/defaultValue.md index 7fce2bb2..8974ecd1 100644 --- a/docs/build/reference/transformer/Value/defaultValue.md +++ b/docs/build/reference/transformer/Value/defaultValue.md @@ -2,10 +2,12 @@ title: "Default Value" description: "Generates a default value, if the input values are empty. Forwards any non-empty values." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Default Value + @@ -45,9 +47,9 @@ Generates a default value, if the input values are empty. Forwards any non-empty The default value to be generated, if input values are empty -- ID: `value` -- Datatype: `string` -- Default Value: `default` +* ID: `value` +* Datatype: `string` +* Default Value: `default` @@ -55,4 +57,4 @@ The default value to be generated, if input values are empty ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/emptyValue.md b/docs/build/reference/transformer/Value/emptyValue.md index c301f231..557072aa 100644 --- a/docs/build/reference/transformer/Value/emptyValue.md +++ b/docs/build/reference/transformer/Value/emptyValue.md @@ -2,10 +2,12 @@ title: "Empty value" description: "Generates an empty value." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Empty value + @@ -19,4 +21,4 @@ Generates an empty value. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/inputHash.md b/docs/build/reference/transformer/Value/inputHash.md index 04ce371f..df64ff7e 100644 --- a/docs/build/reference/transformer/Value/inputHash.md +++ b/docs/build/reference/transformer/Value/inputHash.md @@ -2,10 +2,12 @@ title: "Input hash" description: "Calculates the hash sum of the input values. Generates a single hash sum for all input values combined." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Input hash + @@ -20,10 +22,10 @@ This operator supports using different hash algorithms from the [Secure Hash Alg --- **Example 1:** -* Input values: +- Input values: 1. `[input value]` -* Returns: `[f708c2afff0ed197e8551c4dd549ee5b848e0b407106cbdb8e451c8cd1479362]` +- Returns: `[f708c2afff0ed197e8551c4dd549ee5b848e0b407106cbdb8e451c8cd1479362]` @@ -44,4 +46,4 @@ The hash algorithm to be used. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/randomNumber.md b/docs/build/reference/transformer/Value/randomNumber.md index 01c4f222..6c0e2f85 100644 --- a/docs/build/reference/transformer/Value/randomNumber.md +++ b/docs/build/reference/transformer/Value/randomNumber.md @@ -2,10 +2,12 @@ title: "Random number" description: "Generates a set of random numbers." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Random number + @@ -59,4 +61,4 @@ The maximum number of values to generate in each set. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/readParameter.md b/docs/build/reference/transformer/Value/readParameter.md index fadc0d8e..4bc2b579 100644 --- a/docs/build/reference/transformer/Value/readParameter.md +++ b/docs/build/reference/transformer/Value/readParameter.md @@ -2,10 +2,12 @@ title: "Read parameter" description: "Reads a parameter from a Java Properties file." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # Read parameter + @@ -39,4 +41,4 @@ The name of the parameter. ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/Value/uuid.md b/docs/build/reference/transformer/Value/uuid.md index 0738b722..aeb17ebf 100644 --- a/docs/build/reference/transformer/Value/uuid.md +++ b/docs/build/reference/transformer/Value/uuid.md @@ -2,10 +2,12 @@ title: "UUID" description: "Generates UUIDs. If no input value is provided, a random UUID (type 4) is generated using a cryptographically strong pseudo random number generator. If input values are provided, a name-based UUID (type 3) is generated for each input value. Each input value will generate a separate UUID. For building a UUID from multiple inputs, the Concatenate operator can be used." icon: octicons/cross-reference-24 -tags: +tags: - TransformOperator --- + # UUID + @@ -46,4 +48,4 @@ Each input value will generate a separate UUID. For building a UUID from multipl ## Advanced Parameter -`None` \ No newline at end of file +`None` diff --git a/docs/build/reference/transformer/index.md b/docs/build/reference/transformer/index.md index 66c6a116..aec4bae2 100644 --- a/docs/build/reference/transformer/index.md +++ b/docs/build/reference/transformer/index.md @@ -5,7 +5,9 @@ tags: - Build - Reference --- + # Transformers + Transform operators transform a one or more sequences of string values to a sequence of string values. @@ -71,7 +73,8 @@ Transform operators transform a one or more sequences of string values to a sequ | [Evaluate template](Template/TemplateTransformer.md) | Template | Evaluates a template. Input values can be addressed using the variables 'input1', 'input2', etc. Global variables are available in the 'global' scope, e.g., 'global.myVar'. | | [Even](Excel/Excel_EVEN.md) | Excel | Excel EVEN(number): Rounds the given number up to the nearest even integer. | | [Exact](Excel/Excel_EXACT.md) | Excel | Excel EXACT(text_1; text_2): Compares two text strings and returns TRUE if they are identical. This function is case- sensitive. Text_1 is the first text to compare. Text_2 is the second text to compare. | - | [Excel map](Replace/excelMap.md) | Replace | Replaces values based on a map of values read from a file in Open XML format (XLSX). The XLSX file may contain several sheets of the form: ``` mapFrom,mapTo , ``` and more. An empty string can be created in Excel and alternatives by inserting `=""` in the input line of a cell. If there are multiple values for a single key, all values will be returned for the given key. Note that the mapping table will be cached in memory. If the Excel file is updated (even while transforming), the map will be reloaded within seconds. | + + | [Excel map](Replace/excelMap.md) | Replace | Replaces values based on a map of values read from a file in Open XML format (XLSX). The XLSX file may contain several sheets of the form: ```mapFrom,mapTo ,``` and more. An empty string can be created in Excel and alternatives by inserting `=""` in the input line of a cell. If there are multiple values for a single key, all values will be returned for the given key. Note that the mapping table will be cached in memory. If the Excel file is updated (even while transforming), the map will be reloaded within seconds. | | [Exp](Excel/Excel_EXP.md) | Excel | Excel EXP(number): Returns e raised to the power of the given number. | | [Extract physical quantity](Numeric/extractPhysicalQuantity.md) | Numeric | Extracts physical quantities, such as length or weight values. Values are expected to be formatted as `{Number}{UnitPrefix}{Symbol}` and are converted to the base unit. | | [Fact](Excel/Excel_FACT.md) | Excel | Excel FACT(number): Returns the factorial of the given number. | @@ -83,7 +86,7 @@ Transform operators transform a one or more sequences of string values to a sequ | [Fix URI](Normalize/uriFix.md) | Normalize | Generates valid absolute URIs from the given values. Already valid absolute URIs are left untouched. | | [Floor](Excel/Excel_FLOOR.md) | Excel | Excel FLOOR(number; significance; mode): Rounds the given number down to the nearest multiple of significance. Significance is the value to whose multiple of ten the number is to be rounded down (.01, .1, 1, 10, etc.). Mode is an optional value. If it is indicated and non-zero and if the number and significance are negative, rounding up is carried out based on that value. | | [Forecast](Excel/Excel_FORECAST.md) | Excel | Excel FORECAST(value; data_Y; data_X): Extrapolates future values based on existing x and y values. Value is the x value, for which the y value of the linear regression is to be returned. Data_Y is the array or range of known y's. Data_X is the array or range of known x's. Does not work for exponential functions. | - | [Format number](Numeric/formatNumber.md) | Numeric | Formats a number according to a user-defined pattern. The pattern syntax is documented at: https://docs.oracle.com/javase/8/docs/api/java/text/DecimalFormat.html | + | [Format number](Numeric/formatNumber.md) | Numeric | Formats a number according to a user-defined pattern. The pattern syntax is documented at: | | [Fv](Excel/Excel_FV.md) | Excel | Excel FV(rate; NPER; PMT; PV; type): Returns the future value of an investment based on periodic, constant payments and a constant interest rate. Rate is the periodic interest rate. NPER is the total number of periods. PMT is the annuity paid regularly per period. PV (optional) is the present cash value of an investment. Type (optional) defines whether the payment is due at the beginning (1) or the end (0) of a period. | | [Geomean](Excel/Excel_GEOMEAN.md) | Excel | Excel GEOMEAN(number_1; number_2; ... number_30): Returns the geometric mean of a sample. Number_1; number_2; ... number_30 are numerical arguments or ranges that represent a random sample. | | [Get value by index](Sequence/getValueByIndex.md) | Sequence | Returns the value found at the specified index. Fails or returns an empty result depending on failIfNoFound is set or not. Please be aware that this will work only if the data source supports some kind of ordering like XML or JSON. This is probably not a good idea to do with RDF models. If emptyStringToEmptyResult is true then instead of a result with an empty String, an empty result is returned. | diff --git a/docs/build/snowflake-tutorial/index.md b/docs/build/snowflake-tutorial/index.md index fa7e1828..11a5bcc2 100644 --- a/docs/build/snowflake-tutorial/index.md +++ b/docs/build/snowflake-tutorial/index.md @@ -22,48 +22,48 @@ This integration enables organizations to make informed decisions, improve their This tutorial contains the following step-by-step instructions to connect the Snowflake data-warehouse with eccenca corporate memory: -- [1. Create a database in Snowflake](#1-create-a-database-in-snowflake) -- [2. Create a project in eccenca Corporate Memory](#2-create-a-project-in-eccenca-corporate-memory) -- [3. Create a transformation to build mapping rules](#3-create-a-transformation-to-build-mapping-rules) -- [4. Create a knowledge graph](#4-create-a-knowledge-graph) +- [1. Create a database in Snowflake](#1-create-a-database-in-snowflake) +- [2. Create a project in eccenca Corporate Memory](#2-create-a-project-in-eccenca-corporate-memory) +- [3. Create a transformation to build mapping rules](#3-create-a-transformation-to-build-mapping-rules) +- [4. Create a knowledge graph](#4-create-a-knowledge-graph) ## Sample material The following material is used in this tutorial, you should download the files and have them at hand throughout the tutorial: -- The product data vocabulary [products_vocabulary.nt](products_vocabulary.nt) +- The product data vocabulary [products_vocabulary.nt](products_vocabulary.nt) ## 1. Create a database in Snowflake -- Login to Snowflake enter the **username** and **password**, then click on **Sign in**. +- Login to Snowflake enter the **username** and **password**, then click on **Sign in**. ![image](snowflake-login.png){ width="50%" class="bordered" } -- Click on **Database** on the left side of the page. +- Click on **Database** on the left side of the page. ![image](snowflake-click_on_database.png){ class="bordered" } -- Click on **+Database** on the right side of the page. +- Click on **+Database** on the right side of the page. ![image](snowflake-add-database.png){ class="bordered" } -- Type the database name **Product**, then click on **Create**. +- Type the database name **Product**, then click on **Create**. ![image](snowflake-product-name-database.png){ width="50%" class="bordered" } -- Click on database **product**, then click on **+Schema** on the right side of the page. +- Click on database **product**, then click on **+Schema** on the right side of the page. ![image](snowflake-dd-schema.png){ class="bordered" } -- Type the schema name **products_vocabulary** and click on **Create**. +- Type the schema name **products_vocabulary** and click on **Create**. ![image](snowflake-schema-name.png){ width="50%" class="bordered" } -- Click on scheme **products_vocabulary** on the left side of the page then click on **Create** on the right side of the page, then click on **Table**, then select then **Standard**. +- Click on scheme **products_vocabulary** on the left side of the page then click on **Create** on the right side of the page, then click on **Table**, then select then **Standard**. ![image](snowflake-click-on-standard.png){ class="bordered" } -- Click on schema name **products_vocabulary** on the left side of the page and type the **sql query** for creating a table in the center, then click on **Run** on the right side of the page. +- Click on schema name **products_vocabulary** on the left side of the page and type the **sql query** for creating a table in the center, then click on **Run** on the right side of the page. You can create the _PRODUCT_ table with the following SQL query: @@ -80,7 +80,7 @@ The following material is used in this tutorial, you should download the files a ) ; ``` -- Type or copy the **SQL** query for creating a database in the table that is created, then click on **Run**. +- Type or copy the **SQL** query for creating a database in the table that is created, then click on **Run**. ![image](snowflake-sql-for-table.png){ class="bordered" } @@ -1103,29 +1103,29 @@ Here you can populate some test data with the following SQL query: ## 2. Create a project in eccenca Corporate Memory -- Click on **Create** on the right side of the page. +- Click on **Create** on the right side of the page. ![image](snowflake-create-project.png){ class="bordered" } -- Click on Project, then click on **Add**. +- Click on Project, then click on **Add**. ![image](snowflake-add-project.png){ class="bordered" } -- Type the project name **product** in the title field, then click on **Create**. +- Type the project name **product** in the title field, then click on **Create**. ![image](snowflake-project-name.png){ class="bordered" } -- Click on **Create** on the right side of the page. +- Click on **Create** on the right side of the page. ![image](snowflake-create.png){ class="bordered" } -- Click on **Snowflake SQL endpoint**, then click on **Add**. +- Click on **Snowflake SQL endpoint**, then click on **Add**. ![image](snowflake-jdbc.png){ class="bordered" } -- Type the name **Product table (JDBC)** in the label field. +- Type the name **Product table (JDBC)** in the label field. -- In the **CONNECTION** section populate the required details. +- In the **CONNECTION** section populate the required details. ![image](snowflake-connection.png){ class="bordered" } @@ -1141,7 +1141,7 @@ Here you can populate some test data with the following SQL query: - Select your **Schema**, `products_vocabulary` in this case -- In the **READ** section specify **Source query** as +- In the **READ** section specify **Source query** as ```sparql SELECT * from product @@ -1149,15 +1149,15 @@ Here you can populate some test data with the following SQL query: ![image](snowflake-query-source.png){ class="bordered" } -- Select the **Query strategy** as **Execute the given source query. No paging or virtual query**. +- Select the **Query strategy** as **Execute the given source query. No paging or virtual query**. ![image](snowflake-query-strategy.png){ class="bordered" } -- Select the **Write strategy** as **An exception will be thrown, if the table already exists.** +- Select the **Write strategy** as **An exception will be thrown, if the table already exists.** ![image](snowflake-write-stategy.png){ class="bordered" } -- Then click on **Create**. +- Then click on **Create**. !!! success "Step Result" @@ -1167,52 +1167,52 @@ Here you can populate some test data with the following SQL query: ## 3. Create a transformation to build mapping rules -- Click on **Create** on the right side of the page. +- Click on **Create** on the right side of the page. ![image](snowflake-click-on-create.png){ class="bordered" } -- Click on **Transform** on the left side of the page, then on **Transform** in the centre of the page, then click on **Add**. +- Click on **Transform** on the left side of the page, then on **Transform** in the centre of the page, then click on **Add**. ![image](snowflake-transformation.png){ class="bordered" } -- Type the name **product** in the **Label** field, in the **INPUT TASK Dataset** select **Product Table (JDBC)** and in the **Type** field select **table**. +- Type the name **product** in the **Label** field, in the **INPUT TASK Dataset** select **Product Table (JDBC)** and in the **Type** field select **table**. ![image](snowflake-trans-connect.png){ class="bordered" } ![image](snowflake-type-table.png){ class="bordered" } -- In the **Output** dataset field select **product graph**, then click on **Create** . +- In the **Output** dataset field select **product graph**, then click on **Create** . ![image](snowflake-output-gp.png){ class="bordered" } -- Click on **Mapping**, then click on **Edit**. +- Click on **Mapping**, then click on **Edit**. ![image](snowflake-click-on-mapping.png){ class="bordered" } -- For the target entity select **Product (pv:product)**. +- For the target entity select **Product (pv:product)**. ![image](snowflake-target-entity.png){ class="bordered" } -- Click on **create custom pattern**. +- Click on **create custom pattern**. ![image](snowflake-custom-pattern.png){ class="bordered" } -- Type the URI pattern as ``. +- Type the URI pattern as ``. You can use either company.org or company.com as per your requirement. Then type the label name as **product** and then click on **save**. ![image](snowflake-uri-pattern.png){ class="bordered" } -- Click on **+Icon**, then select the **Add value mapping**. +- Click on **+Icon**, then select the **Add value mapping**. ![image](snowflake-add-value.png){ class="bordered" } -- Select the **target property** according to transformation requirements, for example name, id, etc., then select the **value path** according to the target property as the product name, product id etc. +- Select the **target property** according to transformation requirements, for example name, id, etc., then select the **value path** according to the target property as the product name, product id etc. This step will help in mapping the data from the source to the target property. ![image](snowflake-target-property.png){ class="bordered" } -- Type the label name **product name**, then click on **Save**. +- Type the label name **product name**, then click on **Save**. ![image](snowflake-trans-label.png){ class="bordered" } @@ -1238,25 +1238,25 @@ Here you can populate some test data with the following SQL query: Suggestions generated are based on vocabulary which describes the data in the CSV files: [products_vocabulary.nt](products_vocabulary.nt) -- **Tick** the box to select the suggestions to be added, then click on **Add**. +- **Tick** the box to select the suggestions to be added, then click on **Add**. ![image](snowflake-tick.png){ class="bordered" } ## 4. Create a knowledge graph -- Click on **Create** on the right side of the page. +- Click on **Create** on the right side of the page. ![image](snowflake-trans-result.png){ class="bordered" } -- Select **Knowledge Graph**, then click on **Add**. +- Select **Knowledge Graph**, then click on **Add**. ![image](snowflake-kg-graph.png){ class="bordered" } -- Select the **target project** from the drop down menu as **product**. +- Select the **target project** from the drop down menu as **product**. ![image](snowflake-graph-target.png){ class="bordered" } -- Type **product graph** in the label field, then enter the **graph URI** in the Graph field, then click on **Create**. +- Type **product graph** in the label field, then enter the **graph URI** in the Graph field, then click on **Create**. ![image](snowflake-graph-uri.png){ class="bordered" } diff --git a/docs/build/spark/index.md b/docs/build/spark/index.md index fcb15dfa..1fad411c 100644 --- a/docs/build/spark/index.md +++ b/docs/build/spark/index.md @@ -20,11 +20,11 @@ The documentation is structured in two parts: The main data processing use-cases of Apache Spark are: -* data loading, -* SQL queries, -* machine learning, -* streaming, -* graph processing. +- data loading, +- SQL queries, +- machine learning, +- streaming, +- graph processing. Additionally, there are other functionalities stemming from hundreds of plugins. diff --git a/docs/build/tutorial-how-to-link-ids-to-osint/define-the-interfaces/index.md b/docs/build/tutorial-how-to-link-ids-to-osint/define-the-interfaces/index.md index 2a3029c1..23fe2db7 100644 --- a/docs/build/tutorial-how-to-link-ids-to-osint/define-the-interfaces/index.md +++ b/docs/build/tutorial-how-to-link-ids-to-osint/define-the-interfaces/index.md @@ -9,31 +9,31 @@ In this tutorial, we will show how to (1) define the information available in an In the previous tutorial, we have defined the use cases in contact with the humans, ie. analysts (see figure 1) -

- -

- -*Figure 1. We need to imagine an interface where analysts can list the IoCs during the incident and read all their documentations.* +
+![Figure 1. We need to imagine an interface where analysts can list the IoCs during the incident and read all their documentations.](use_cases_with_interfaces.png) +
Figure 1. We need to imagine an interface where analysts can list the IoCs during the incident and read all their documentations.
+
The classic Splunk interface is a set of panels, like "static table" panel. This table panel can show a table of cells and also one cell with a text via in input a Splunk Search Processing Language (SPL). With the plugin [Linked Data App](../link-IDS-event-to-KG/eccenca_commands.tar.gz) ([tutorial page](../link-IDS-event-to-KG/index.md)), we can insert a SPARQL query and select the part of your knowledge graph to print (figure 2). -![](demo-splunk-with-kg-edit.png) - -*Figure 2. An analyst can insert a SPARQL query with Splunk token in input of one "static table" panel of his dashboard with the plugin "Linked Data App"* +
+![Figure 2. An analyst can insert a SPARQL query with Splunk token in input of one "static table" panel of his dashboard with the plugin "Linked Data App"](demo-splunk-with-kg-edit.png) +
Figure 2. An analyst can insert a SPARQL query with Splunk token in input of one "static table" panel of his dashboard with the plugin "Linked Data App"
+
The first dashboard to do for our use cases is the list of IoCs with classic SPL queries of analysts via a static table and allow the analyst to select one IoC via a click in the table. The dashboard with this selected row can save the ID of IoC in a global variable for the other panels in the same dashboard ([a Splunk token](https://docs.splunk.com/Documentation/Splunk/9.0.5/Viz/tokens)). When this variable (Splunk token) is changed by the user, Splunk is able to recalculate automatically the queries with this variable in the other static tables. So with this mechanism, we can print the details in the knowledge graph (with SPARQL queries) and the IoC statistics in the Splunk indexes (with SPL queries) around of one selected IoC. With these knowledge about Splunk dashboard, we proposed to analysts a first naive interface in the figure 3. -

- -

- -*Figure 3. Imagine the expected Splunk dashboard with its interaction* +
+![Figure 3. Imagine the expected Splunk dashboard with its interaction](interface.png) +
Figure 3. Imagine the expected Splunk dashboard with its interaction
+
Here, the figure 3 is nice but before this first schema during the project, there are a lot of shemas and all were minimalist and ugly often only on a whiteboard. This type schema before the technical feasibility is only to validate the objective with the analysts before starting the development. During the technical feasibility, we can decrease/increase step-by-step your objectives to show finally a first result in figure 4 in a real dashboard. -![](../link-IDS-event-to-KG/demo_ld_without_html.png) - -*Figure 4. First interface with only SPARQL queries in SPLUNK static tables.* +
+![Figure 4. First interface with only SPARQL queries in SPLUNK static tables.](../link-IDS-event-to-KG/demo_ld_without_html.png) +
Figure 4. First interface with only SPARQL queries in SPLUNK static tables.
+
## Technical feasibility with the available information @@ -42,9 +42,10 @@ In this tutorial, we learn to use Corporate Memory of Eccenca to transform these After research and one meeting with analysts, we have chosen the datasets of Mitre Attack, the datasets of IoC rules (Sigma and Hayabusa) in Github and of course, the IoCs data already in the Splunk indexes. -![](alert_data_of_ioc.png) - -*Figure 5. Define the information available in alerts of IDS and in OSINT to link these information.* +
+![Figure 5. Define the information available in alerts of IDS and in OSINT to link these information.](alert_data_of_ioc.png) +
Figure 5. Define the information available in alerts of IDS and in OSINT to link these information.
+
The Splunk indexes of IoCs are selected by the analysts in the dashboard via the component [multiselect input](https://docs.splunk.com/Documentation/Splunk/9.0.5/Viz/FormEditor#Multiselect) in the form part of dashboard (the form part inits other Splunk tokens). We have choosen the IDs to link these data and the figure 5 resumes how we are going to link these data via Corporate Memory of Eccenca. @@ -56,9 +57,9 @@ A user can clone any dashboard before modifying it. For example, in our dashboard, you can find: -- the root element `form`, -- the definition of input component to select the Splunk indexes by the user and -- the table panel to execute a SPL query and show the result in a table +- the root element `form`, +- the definition of input component to select the Splunk indexes by the user and +- the table panel to execute a SPL query and show the result in a table ```xml
@@ -158,19 +159,24 @@ WHERE { During our project, we have implemented the SPARQL command necessary to execute a SPARQL query in a SPL query but also several scripts to extend the panels of dashboard. For example, these are problems to print a HTML text and open a external Web page in a dashboard. Before starting a knowledge graph, we need to know if we have to work with a specific syntax in output for Splunk. So, we have searched the simplest solution to print the HTML literal in our knowledge graph with their links. We found it and implemented simple Javascript scripts to resolve these problems. These scripts are imported via the header of dashboard XML and called in the XML of static table panel. You can see the final dashboard with the Mitre description in HTML (the Mitre in these datasets uses Markdown but we show how convert Markdown link to HTML). We give you these scripts in your [Linked Data App](../link-IDS-event-to-KG/eccenca_commands.tar.gz) ([tutorial page](../link-IDS-event-to-KG/index.md)). -![](../link-IDS-event-to-KG/demo_ld.png) - -*Figure 6. With an extern Javascript script, static tables support HTML and the user can open.* +
+![Figure 6. With an extern Javascript script, static tables support HTML and the user can open.](../link-IDS-event-to-KG/demo_ld.png) +
Figure 6. With an extern Javascript script, static tables support HTML and the user can open.
+
## Conclusion: starting to specify the necessary RDF models for these interfaces With the interfaces, the available data and their links in head, the analyst can now imagine the necessary RDF models of concepts (for example, figure 7 and 8) in his future knowledge graph to generate expected dashboards. These RDF models evolve at the same time as the interfaces (forever) and according to future RDF standards of Cyber world. With Corporate Memory, after each evolution of your models, you can rebuild your knowledge graph "from scratch" when you want. Several RDF models of different versions can exist in your knowledge graphs, so you can update progressively your dashboards without service interruption of old dashboards. -![](../lift-data-from-STIX-2.1-data-of-mitre-attack/rdf-model-course-of-action.png) -*Figure 7. RDF model of Mitre concept "course of action" in our future knowledge graph.* +
+![Figure 7. RDF model of Mitre concept "course of action" in our future knowledge graph.](../lift-data-from-STIX-2.1-data-of-mitre-attack/rdf-model-course-of-action.png) +
Figure 7. RDF model of Mitre concept "course of action" in our future knowledge graph.
+
-![](../lift-data-from-YAML-data-of-hayabusa-sigma/23-1-rdf-model-rule.png) -*Figure 8. RDF model of concept "IoC Rule" in our future knowledge graph.* +
+![Figure 8. RDF model of concept "IoC Rule" in our future knowledge graph.](../lift-data-from-YAML-data-of-hayabusa-sigma/23-1-rdf-model-rule.png) +
Figure 8. RDF model of concept "IoC Rule" in our future knowledge graph.
+
--- diff --git a/docs/build/tutorial-how-to-link-ids-to-osint/define-the-need/index.md b/docs/build/tutorial-how-to-link-ids-to-osint/define-the-need/index.md index 035fc148..903e7e13 100644 --- a/docs/build/tutorial-how-to-link-ids-to-osint/define-the-need/index.md +++ b/docs/build/tutorial-how-to-link-ids-to-osint/define-the-need/index.md @@ -75,9 +75,10 @@ In this tutorial, we study only this first result: To achieve such savings we aggregate all links of sources and references about alerts in the Security information and event management (SIEM) in a knowledge graph. Analyst are able to read the Mitre information directly in his timeline (e.g. in SPLUNK) and to access all references about an alert from this central place. - ![](slide_result_expected.png) - - *Figure 1. Example of expected results for analysts during the task to understand the meaning and relevance of new alerts in their IDS.* +
+ ![Figure 1. Example of expected results for analysts during the task to understand the meaning and relevance of new alerts in their IDS.](slide_result_expected.png) +
Figure 1. Example of expected results for analysts during the task to understand the meaning and relevance of new alerts in their IDS.
+
When we know the waited results, we can imagine the necessary use cases. @@ -85,9 +86,10 @@ When we know the waited results, we can imagine the necessary use cases. We know the final need, the expected results and the limits of knowledge graph with the Linked Data technologies in Splunk. So, we can define the probable use cases to implement and all the actors who will interact with these use cases. -![](basic_use_cases.png) - -*Figure 2. UML use cases to resolve this basic need and several use cases with Wikidata to show the interoperability of knowledge graphs with the Linked data technologies. Each bubble in this type of schema is a use case.* +
+![Figure 2. UML use cases to resolve this basic need and several use cases with Wikidata to show the interoperability of knowledge graphs with the Linked data technologies. Each bubble in this type of schema is a use case.](basic_use_cases.png) +
Figure 2. UML use cases to resolve this basic need and several use cases with Wikidata to show the interoperability of knowledge graphs with the Linked data technologies. Each bubble in this type of schema is a use case.
+
With a simple UML schema of use cases, you can delimited each use case, their priorities and their tasks for the next step, ie. specify the essential interfaces to limit the complexity of future RDF graph. @@ -110,13 +112,13 @@ Another result of this project was to resolve this other need: For the moment, we are searching the best way to resolve this need but a demonstrator to manage several investigations in the same knowledge graphs is available with several examples of dasboards in the Splunk app "[Investigate lateral movements with a knowledge graph](../link-IDS-event-to-KG-via-cmem/eccenca_poc_investigate.tar.gz)" ([tutorial page](../link-IDS-event-to-KG/index.md)). This need is for advanced users of Corporate Memory and it may be proposed in a future tutorial. -![](advanced_use_cases.png) - -*Figure 3. UML use cases to resolve this avanced need.* +
+![Figure 3. UML use cases to resolve this avanced need.](advanced_use_cases.png) +
Figure 3. UML use cases to resolve this avanced need.
+
--- Tutorial: [how to link Intrusion Detection Systems (IDS) to Open-Source INTelligence (OSINT)](../index.md) Next chapter: [Specify the dashboards of use cases before the RDF models](../define-the-interfaces/index.md) - diff --git a/docs/build/tutorial-how-to-link-ids-to-osint/index.md b/docs/build/tutorial-how-to-link-ids-to-osint/index.md index f3c56e55..ff046ff0 100644 --- a/docs/build/tutorial-how-to-link-ids-to-osint/index.md +++ b/docs/build/tutorial-how-to-link-ids-to-osint/index.md @@ -44,4 +44,4 @@ For the part "Link IDS event to a knowledge graph in dashboards", you need to ha --- -Next chapter: [Define the need, the expected result and the use cases](./define-the-need/index.md) +Next chapter: [Define the need, the expected result and the use cases](define-the-need/index.md) diff --git a/docs/build/tutorial-how-to-link-ids-to-osint/lift-data-from-STIX-2.1-data-of-mitre-attack/index.md b/docs/build/tutorial-how-to-link-ids-to-osint/lift-data-from-STIX-2.1-data-of-mitre-attack/index.md index ef1b1647..8d176897 100644 --- a/docs/build/tutorial-how-to-link-ids-to-osint/lift-data-from-STIX-2.1-data-of-mitre-attack/index.md +++ b/docs/build/tutorial-how-to-link-ids-to-osint/lift-data-from-STIX-2.1-data-of-mitre-attack/index.md @@ -6,12 +6,11 @@ MITRE ATT&CK is a globally-accessible knowledge base of adversary tactics and te The MITRE ATT&CK datasets in STIX 2.1 JSON collections are here: -* [enterprise-attack.json](https://github.com/mitre-attack/attack-stix-data/tree/master/enterprise-attack/enterprise-attack.json){target=_blank} -* [mobile-attack.json](https://github.com/mitre-attack/attack-stix-data/blob/master/mobile-attack/mobile-attack.json){target=_blank} -* [ics-attack.json](https://github.com/mitre-attack/attack-stix-data/blob/master/ics-attack/ics-attack.json){target=_blank} +* [enterprise-attack.json](https://github.com/mitre-attack/attack-stix-data/tree/master/enterprise-attack/enterprise-attack.json){target=_blank} +* [mobile-attack.json](https://github.com/mitre-attack/attack-stix-data/blob/master/mobile-attack/mobile-attack.json){target=_blank} +* [ics-attack.json](https://github.com/mitre-attack/attack-stix-data/blob/master/ics-attack/ics-attack.json){target=_blank} -[Structured Threat Information Expression (STIX™)]( -https://oasis-open.github.io/cti-documentation/stix/intro.html) is a language and serialization format used to exchange cyber threat intelligence (CTI). +[Structured Threat Information Expression (STIX™)](https://oasis-open.github.io/cti-documentation/stix/intro.html) is a language and serialization format used to exchange cyber threat intelligence (CTI). The "ontology" of MITRE ATT&CK with STIX is here: [https://github.com/mitre/cti/blob/master/USAGE.md](https://github.com/mitre/cti/blob/master/USAGE.md) @@ -51,9 +50,9 @@ For each type of dataset, you can create an new project with all the tools neces Create a new project, reproduce the demonstration in the following video: -* Title: MITRE ATT&CK® +* Title: MITRE ATT&CK® -* Description: MITRE ATT&CK® is a globally-accessible knowledge base of adversary tactics and techniques based on real-world observations. +* Description: MITRE ATT&CK® is a globally-accessible knowledge base of adversary tactics and techniques based on real-world observations. ![](23-1-create-project.gif) @@ -63,22 +62,22 @@ MITRE ATT&CK® has 3 domains: [Entreprise](https://attack.mitre.org/techniques/e Each domain dataset is saved in GitHub: -* [enterprise-attack.json](https://github.com/mitre-attack/attack-stix-data/tree/master/enterprise-attack/enterprise-attack.json) -* [mobile-attack.json](https://github.com/mitre-attack/attack-stix-data/blob/master/mobile-attack/mobile-attack.json) -* [ics-attack.json](https://github.com/mitre-attack/attack-stix-data/blob/master/ics-attack/ics-attack.json) +* [enterprise-attack.json](https://github.com/mitre-attack/attack-stix-data/tree/master/enterprise-attack/enterprise-attack.json) +* [mobile-attack.json](https://github.com/mitre-attack/attack-stix-data/blob/master/mobile-attack/mobile-attack.json) +* [ics-attack.json](https://github.com/mitre-attack/attack-stix-data/blob/master/ics-attack/ics-attack.json) 1. Download these 3 files 2. Create for each JSON file, a JSON dataset: -![](23-1-import-JSON.gif) + ![](23-1-import-JSON.gif) -!!! Tip + !!! Tip - Give a short name at each dataset/transformer/etc in Corporate Memory to recognize it easily in the workflow view. For example, we will use "MA Entreprise (JSON)" like label and "MITRE ATT&CK® Entreprise dataset STIX 2.1" like description for the Entreprise dataset and so "MA Mobile (JSON)" for Mobile, "MA ICS (JSON)" for ICS, etc. + Give a short name at each dataset/transformer/etc in Corporate Memory to recognize it easily in the workflow view. For example, we will use "MA Entreprise (JSON)" like label and "MITRE ATT&CK® Entreprise dataset STIX 2.1" like description for the Entreprise dataset and so "MA Mobile (JSON)" for Mobile, "MA ICS (JSON)" for ICS, etc. -!!! Success + !!! Success - Now, you can see these JSON datasets in Corporate Memory: - ![](23-1-import-JSON-result.png) + Now, you can see these JSON datasets in Corporate Memory: + ![](23-1-import-JSON-result.png) ### Create named graphs @@ -114,18 +113,18 @@ Create one RDF dataset for each Mitre dataset: 3. Put a URI of named graph 4. Enable "Clear graph before workflow execution" -![](23-1-create-RDF-dataset.gif) + ![](23-1-create-RDF-dataset.gif) -!!! Success + !!! Success - Now, you can see these RDF datasets in Corporate Memory: - ![](23-1-create-RDF-dataset-result.png) + Now, you can see these RDF datasets in Corporate Memory: + ![](23-1-create-RDF-dataset-result.png) -!!! Tip + !!! Tip - The consequence of the option "Clear graph before workflow execution" is the named graph will be deleted (with all its triples) before receiving new triples when you use this named graph like an output in a workflow and also in the transformer task (in the next step). + The consequence of the option "Clear graph before workflow execution" is the named graph will be deleted (with all its triples) before receiving new triples when you use this named graph like an output in a workflow and also in the transformer task (in the next step). - This option is to use only for the graphs which will generate automatically by Corporate Memory. + This option is to use only for the graphs which will generate automatically by Corporate Memory. ### Create a transformer @@ -141,7 +140,7 @@ Create one RDF dataset for each Mitre dataset: Here, you will create all classes and attributes necessary in your use case case. Not more, not less. So, we are adding each STIX object in your knowledge base with its STIX type, its label, its description and its references. Each reference can have an url, a label, a description and an external ID, like Mitre ID or CAPEC ID. - In UML, you can represent your targeted model like that: here a RDF model to describe an instance of type "course-of-action" in MITRE ATT&CK. (you can download the [File drawio of schemas](./RDF_model_and_pattern.drawio)) + In UML, you can represent your targeted model like that: here a RDF model to describe an instance of type "course-of-action" in MITRE ATT&CK. (you can download the [File drawio of schemas](RDF_model_and_pattern.drawio)) ![RDF model to describe an instance of type "course-of-action" in MITRE ATT&CK](rdf-model-course-of-action.png) @@ -159,253 +158,253 @@ Create one RDF dataset for each Mitre dataset: 1. Create the prefix of your vocabulary: - ```turtle - prefix ctia: - ``` + ```turtle + prefix ctia: + ``` -![](23-1-create-prefix.gif) + ![](23-1-create-prefix.gif) 2. Create the (Mitre) STIX 2.1 transformer -This transformer will be a component of your worflow. You could reuse it in several workflows in other projects. To create a new transformer, you need to give a: + This transformer will be a component of your worflow. You could reuse it in several workflows in other projects. To create a new transformer, you need to give a: -* Label: STIX 2.1 transformer -* Input: MA Entreprise (JSON) -* Output: MA Entreprise (knowledge graph) + * Label: STIX 2.1 transformer + * Input: MA Entreprise (JSON) + * Output: MA Entreprise (knowledge graph) -![](23-1-create-transformer.gif) + ![](23-1-create-transformer.gif) -!!! Tip + !!! Tip - In your use case, there is only this transformer to build this named graph, so there is no consequence on the final knowledge graph when we test this transformer on this graph (automatically cleared after each execution of transformer). However, a good practice is to create a tempory graph in ouput for each transformer, so your final knowledge graph is not affected during the modification of your transformer before executing the workflows with this transformer. In this case, you need to hide this tempory graph of your users. + In your use case, there is only this transformer to build this named graph, so there is no consequence on the final knowledge graph when we test this transformer on this graph (automatically cleared after each execution of transformer). However, a good practice is to create a tempory graph in ouput for each transformer, so your final knowledge graph is not affected during the modification of your transformer before executing the workflows with this transformer. In this case, you need to hide this tempory graph of your users. - You can create a transformer for several syntaxes in input: JSON, XML, CSV, etc. If your format does not exist in Corporate Memory, you can convert your data in JSON before importing this data in Corporate Memory. + You can create a transformer for several syntaxes in input: JSON, XML, CSV, etc. If your format does not exist in Corporate Memory, you can convert your data in JSON before importing this data in Corporate Memory. -!!! Info + !!! Info - STIX gives the possibility to extend its syntaxes. Mitre uses this possibility. So, in theory, if we need to import all the data, we can extend this transformer at all STIX attributes and add the Mitre attributes described in its [documentation](https://github.com/mitre/cti/blob/master/USAGE.md). + STIX gives the possibility to extend its syntaxes. Mitre uses this possibility. So, in theory, if we need to import all the data, we can extend this transformer at all STIX attributes and add the Mitre attributes described in its [documentation](https://github.com/mitre/cti/blob/master/USAGE.md). 3. Study the tree of STIX data -```json -{ - "type": "bundle", - "id": "bundle--19413d5e-67e5-4a48-a4c8-afb06b7954de", - "spec_version": "2.1", - "objects": [ - { - "type": "x-mitre-collection", - "id": "x-mitre-collection--1f5f1533-f617-4ca8-9ab4-6a02367fa019", - "name": "Enterprise ATT&CK", - "description": "ATT&CK for Enterprise provides a knowledge base of real-world adversary behavior targeting traditional enterprise networks. ATT&CK for Enterprise covers the following platforms: Windows, macOS, Linux, PRE, Office 365, Google Workspace, IaaS, Network, and Containers.", - ... - }, - { - "id": "attack-pattern--0042a9f5-f053-4769-b3ef-9ad018dfa298", - "type": "attack-pattern", - "name": "Extra Window Memory Injection", - "description": "Adversaries may inject malicious code..." , - "external_references": [ - { - "source_name": "mitre-attack", - "external_id": "T1055.011", - "url": "https://attack.mitre.org/techniques/T1055/011" - }, - { - "url": "https://msdn.microsoft.com/library/windows/desktop/ms633574.aspx", - "description": "Microsoft. (n.d.). About Window Classes. Retrieved December 16, 2017.", - "source_name": "Microsoft Window Classes" - },... -``` + ```json + { + "type": "bundle", + "id": "bundle--19413d5e-67e5-4a48-a4c8-afb06b7954de", + "spec_version": "2.1", + "objects": [ + { + "type": "x-mitre-collection", + "id": "x-mitre-collection--1f5f1533-f617-4ca8-9ab4-6a02367fa019", + "name": "Enterprise ATT&CK", + "description": "ATT&CK for Enterprise provides a knowledge base of real-world adversary behavior targeting traditional enterprise networks. ATT&CK for Enterprise covers the following platforms: Windows, macOS, Linux, PRE, Office 365, Google Workspace, IaaS, Network, and Containers.", + … + }, + { + "id": "attack-pattern--0042a9f5-f053-4769-b3ef-9ad018dfa298", + "type": "attack-pattern", + "name": "Extra Window Memory Injection", + "description": "Adversaries may inject malicious code..." , + "external_references": [ + { + "source_name": "mitre-attack", + "external_id": "T1055.011", + "url": "https://attack.mitre.org/techniques/T1055/011" + }, + { + "url": "https://msdn.microsoft.com/library/windows/desktop/ms633574.aspx", + "description": "Microsoft. (n.d.). About Window Classes. Retrieved December 16, 2017.", + "source_name": "Microsoft Window Classes" + },… + ``` -To extract STIX objects with its type, its label, its description and its references, we need to navigate via a root object of type "bundle" before touching the STIX objects. Each object has an ID, we suppose unique in all Mitre datasets to generate IRI of all objects. We use your prefix ctia to build the class name and the properties of your RDFS vocabulary. Here, we build the vocabulary of manner agile for your use case because Mitre had not proposed a RDFS vocabulary for its datasets. + To extract STIX objects with its type, its label, its description and its references, we need to navigate via a root object of type "bundle" before touching the STIX objects. Each object has an ID, we suppose unique in all Mitre datasets to generate IRI of all objects. We use your prefix ctia to build the class name and the properties of your RDFS vocabulary. Here, we build the vocabulary of manner agile for your use case because Mitre had not proposed a RDFS vocabulary for its datasets. 4. Create the root object and give it an unique IRI: -* RDF type: ctia:Object -* IRI pattern: + * RDF type: ctia:Object + * IRI pattern: -![](23-1-extract-root-object.gif) + ![](23-1-extract-root-object.gif) -!!! Tip + !!! Tip - You can develop an IRI from scratch in the IRI formula editor, like here or directly in the form and improve it after, if necessary (see an example in the next step). + You can develop an IRI from scratch in the IRI formula editor, like here or directly in the form and improve it after, if necessary (see an example in the next step). - The important is to test the result in the evaluation view. + The important is to test the result in the evaluation view. -!!! Success + !!! Success - During the development of a transformer, you can test your transformation and check all the steps. + During the development of a transformer, you can test your transformation and check all the steps. - ![](23-1-see-steps-during-a-transformation.png) + ![](23-1-see-steps-during-a-transformation.png) 5. Link the sub-objects to their root: -* Value path: objects + * Value path: objects -with their IRI and the property ctia:object: + with their IRI and the property ctia:object: -* RDF property: ctia:object -* RDF type: ctia:Object -* IRI pattern: + * RDF property: ctia:object + * RDF type: ctia:Object + * IRI pattern: -![](23-1-extract-objects.gif) + ![](23-1-extract-objects.gif) -!!! Tip + !!! Tip - The RDFS classes start by an uppercase and the property by a lowercase and apply the camel case notation, if possible. The objective is to create cool IRI, ie. lisible IDs for humans and unique on the Web. + The RDFS classes start by an uppercase and the property by a lowercase and apply the camel case notation, if possible. The objective is to create cool IRI, ie. lisible IDs for humans and unique on the Web. - There are exceptions, like Wikidata which prefers to use a number for their IRI but with a explicit label in all languages. + There are exceptions, like Wikidata which prefers to use a number for their IRI but with a explicit label in all languages. - Moreover, if there is no clear ontology in your domain, the best is to take the name of parameters of the source (here json). So, we will use the property, like `ctia:external_id` with underscore because it's the convention of Mitre in its datasets. If Mitre defines a best RDF ontology, we will modify simply your transformer to respect their new ontology. + Moreover, if there is no clear ontology in your domain, the best is to take the name of parameters of the source (here json). So, we will use the property, like `ctia:external_id` with underscore because it's the convention of Mitre in its datasets. If Mitre defines a best RDF ontology, we will modify simply your transformer to respect their new ontology. -!!! Tip + !!! Tip - We could limit the number of objects to import, if you add conditions in the formula editor with the field "type" of objects, for example. + We could limit the number of objects to import, if you add conditions in the formula editor with the field "type" of objects, for example. 6. Extract now their type, label and description with these properties for example: -* ctia:type - * RDF type: URI - * Via the "value forma editor" create the IRI: `https://github.com/mitre/cti/blob/master/USAGE.md#{type}` -* rdfs:label - * value path: name - * RDF type: String -* ctia:description - * value path: description - * RDF type: String + * ctia:type + * RDF type: URI + * Via the "value forma editor" create the IRI: `https://github.com/mitre/cti/blob/master/USAGE.md#{type}` + * rdfs:label + * value path: name + * RDF type: String + * ctia:description + * value path: description + * RDF type: String -![](23-1-extract-properties.gif) + ![](23-1-extract-properties.gif) -!!! Tip + !!! Tip - STIX type doesn't apply the camel case and doesn't start by an uppercase. We prefers to create a specific property ctia:type for this reason. + STIX type doesn't apply the camel case and doesn't start by an uppercase. We prefers to create a specific property ctia:type for this reason. - You can reuse a vocabulary already in Corporate Memory (like rdfs) but you are also free to develop a new vocabulary on the fly with your prefixes. + You can reuse a vocabulary already in Corporate Memory (like rdfs) but you are also free to develop a new vocabulary on the fly with your prefixes. -!!! Success + !!! Success - When you test your transformer, you can see the future instances in your knowledge graph: - ![](23-1-success-transformer.png) + When you test your transformer, you can see the future instances in your knowledge graph: + ![](23-1-success-transformer.png) 7. At the end of the last step, we saw the dataset uses the syntax of Markdown to define a Web link. In the interface of SPLUNK, we need to use the HTML syntax. Modify the formula for the description with the operator "regex replace". -* Regex: `\[([^\[\]]*)\]\(([^\(\)]*)\)` -* Replace: `$1` + * Regex: `\[([^\[\]]*)\]\(([^\(\)]*)\)` + * Replace: `$1` -![](23-1-regex-replace.gif) + ![](23-1-regex-replace.gif) -!!! Success - In the "value formula editor", you can immediatly check the result of your formula. - ![](23-1-regex-replace.png) + !!! Success + In the "value formula editor", you can immediatly check the result of your formula. + ![](23-1-regex-replace.png) -!!! Tip + !!! Tip - At any moment, you will modify your vocabulary according to your needs that you will find during your development. You need to modify this transformer and relaunch all your workflows which use this transformer. + At any moment, you will modify your vocabulary according to your needs that you will find during your development. You need to modify this transformer and relaunch all your workflows which use this transformer. -!!! Tip + !!! Tip - The regular expression are often necessary in the components of "value formula editor". The website [regex101](https://regex101.com/) will help you to develop and debug the regular expressions. + The regular expression are often necessary in the components of "value formula editor". The website [regex101](https://regex101.com/) will help you to develop and debug the regular expressions. 8. Via the same method, we are linking the references objects to their STIX objects: -* via the property: `ctia:external_references` -* Type: ctia:Reference -* value path: external_references -* IRI of each object: its own URL () + * via the property: `ctia:external_references` + * Type: ctia:Reference + * value path: external_references + * IRI of each object: its own URL () -ctia:Reference object has these properties: + ctia:Reference object has these properties: -* ctia:source_name -* ctia:description -* ctia:url -* ctia:external_id + * ctia:source_name + * ctia:description + * ctia:url + * ctia:external_id -![](23-1-extract-references.gif) + ![](23-1-extract-references.gif) -!!! Tip + !!! Tip - Sometimes, several urls are not correct. You can use the component "Fix URI" to fix the classic problems. + Sometimes, several urls are not correct. You can use the component "Fix URI" to fix the classic problems. - ![](23-2-fix-url.png) + ![](23-2-fix-url.png) -!!! Warning + !!! Warning - When you make a transformer on a dataset, you see quickly the limit of data. For example with Mitre, several references are a set of citations without URL. + When you make a transformer on a dataset, you see quickly the limit of data. For example with Mitre, several references are a set of citations without URL. - ![](23-1-warning-bad-uri.png) + ![](23-1-warning-bad-uri.png) - For example references with this description: `(Citation: Palo Alto menuPass Feb 2017)(Citation: DOJ APT10 Dec 2018)(Citation: District Court of NY APT10 Indictment December 2018)` + For example references with this description: `(Citation: Palo Alto menuPass Feb 2017)(Citation: DOJ APT10 Dec 2018)(Citation: District Court of NY APT10 Indictment December 2018)` - The URL for the majority of citations can be found in the dataset but we need to do a first pass before to link correctly the citations at their URL. + The URL for the majority of citations can be found in the dataset but we need to do a first pass before to link correctly the citations at their URL. - Moreover, we can find also citation directly in the description of several objects but without URL and without their references in their JSON tree. + Moreover, we can find also citation directly in the description of several objects but without URL and without their references in their JSON tree. - Here, it's a simple tutorial. So, we do not try to fix this problem of citations for the moment, but if you want a tutorial to fix it, let me a comment in this page. + Here, it's a simple tutorial. So, we do not try to fix this problem of citations for the moment, but if you want a tutorial to fix it, let me a comment in this page. -!!! Success + !!! Success - To test your transformer, you need to develop one or several SPARQL queries with the RDF pattern which will use in your use case. You are developing this query in the SPARQL editor: + To test your transformer, you need to develop one or several SPARQL queries with the RDF pattern which will use in your use case. You are developing this query in the SPARQL editor: - ```sparql - #Test 1 transformer STIX 2.1 + ```sparql + #Test 1 transformer STIX 2.1 - PREFIX rdfs: - PREFIX ctia: + PREFIX rdfs: + PREFIX ctia: - SELECT - ?title ?description - (GROUP_CONCAT(?link; separator="
") as ?references) - FROM - WHERE { - { - ?resource ctia:type ctia:course-of-action . - } union { - ?resource ctia:type ctia:attack-pattern . - } + SELECT + ?title ?description + (GROUP_CONCAT(?link; separator="
") as ?references) + FROM + WHERE { + { + ?resource ctia:type ctia:course-of-action . + } union { + ?resource ctia:type ctia:attack-pattern . + } - ?resource rdfs:label ?title ; - ctia:description ?description ; - ctia:external_references ?mitre_url . + ?resource rdfs:label ?title ; + ctia:description ?description ; + ctia:external_references ?mitre_url . - ?mitre_url ctia:external_id "T1490" ; - ctia:source_name "mitre-attack" . + ?mitre_url ctia:external_id "T1490" ; + ctia:source_name "mitre-attack" . - OPTIONAL { - ?resource ctia:external_references [ - ctia:url ?reference_url ; - ctia:source_name ?reference_label ; - ctia:description ?reference_description - ] . - BIND( CONCAT("",?reference_label,": ",?reference_description ,"") as ?link) - } - } - GROUP BY ?title ?description - ``` + OPTIONAL { + ?resource ctia:external_references [ + ctia:url ?reference_url ; + ctia:source_name ?reference_label ; + ctia:description ?reference_description + ] . + BIND( CONCAT("",?reference_label,": ",?reference_description ,"") as ?link) + } + } + GROUP BY ?title ?description + ``` - ![](23-1-sparql-query.gif) + ![](23-1-sparql-query.gif) 9. During the building of interfaces, we saw the same MITRE ID of IoC rules is used by the concepts of tactic, mitigation, technique,... In the final interface, we will print properly the label of each concept for the same Mitre ID, like "Technique TXX" or "Mitigation TXX". -!!! Tip + !!! Tip - Moreover, Corporate Memory indexes some specific properties automatically, like rdfs:label. Without this property, it's not easy to find the objects by a search by text. To facilite the research of references, like the mitre id, you are adding the property rdfs:label to reference objects. + Moreover, Corporate Memory indexes some specific properties automatically, like rdfs:label. Without this property, it's not easy to find the objects by a search by text. To facilite the research of references, like the mitre id, you are adding the property rdfs:label to reference objects. -So, we add a new property `rdfs:label` to object `ctia:Reference`. If the reference is not a Mitre ID, we will copy the source_name else we will extract the type of concept in the URL and concat his Mitre ID: + So, we add a new property `rdfs:label` to object `ctia:Reference`. If the reference is not a Mitre ID, we will copy the source_name else we will extract the type of concept in the URL and concat his Mitre ID: -* In the transformer STIX, add the property rdfs:label (type string) to object `ctia:Reference`. + * In the transformer STIX, add the property rdfs:label (type string) to object `ctia:Reference`. -![](rdf-model-course-of-action.png) + ![](rdf-model-course-of-action.png) -* Customize the value of label, like in this RDF model: (try to do this rule alone before to look at this possible response) + * Customize the value of label, like in this RDF model: (try to do this rule alone before to look at this possible response) -![](23-1-extract-rdfslabel.png) + ![](23-1-extract-rdfslabel.png) -!!! Success + !!! Success - You can test the result when you search the Mitre ID via the explorer of knowledge graph "MA Entreprise": + You can test the result when you search the Mitre ID via the explorer of knowledge graph "MA Entreprise": - + ![](23-1-success-extract-rdfslabel.png) ### Create a workflow @@ -414,20 +413,20 @@ You have now a STIX transformer. We are building here a workflow to apply this t 1. Create a workflow with a name, for example "MITRE ATT&CK® workflow" 2. Insert the input JSON dataset 3. Insert the output RDF dataset -3. Insert the transformer -4. Link the three components -5. Execute the workflow to test it -6. Save it +4. Insert the transformer +5. Link the three components +6. Execute the workflow to test it +7. Save it ![](23-1-create-workflow.gif) -7. Do the same operations for the two other datasets. +8. Do the same operations for the two other datasets. -!!! Success + !!! Success - At the end, the workflow looks like that: + At the end, the workflow looks like that: - ![](23-1-success-worflow.png) + ![](23-1-success-worflow.png) ### Create a global named graph @@ -443,69 +442,69 @@ We are showing the "SPARQL tasks", another important feature available in Corpor 1. Create a "Knowledge Graph" dataset (ie, a RDF dataset) - * Label: MITRE ATT&CK® (knowledge graph) - * URI (name of graph): - * Enable "Clear graph before workflow execution" + * Label: MITRE ATT&CK® (knowledge graph) + * URI (name of graph): + * Enable "Clear graph before workflow execution" 2. Create a "SPARQL Update query" task without missing to enable the Jinja Template - * Label: Import graph - -```sparql -PREFIX owl: - -INSERT DATA { - GRAPH $outputProperties.uri("graph") { - $outputProperties.uri("graph") - owl:imports $inputProperties.uri("graph") . - } -} -``` - -!!! Note - - In this query, Jinja replace $outputProperties.uri("graph") and $inputProperties.uri("graph") according to our workflow so the final code executed of this query is, for example: + * Label: Import graph ```sparql PREFIX owl: INSERT DATA { - GRAPH { - - owl:imports . + GRAPH $outputProperties.uri("graph") { + $outputProperties.uri("graph") + owl:imports $inputProperties.uri("graph") . } } ``` -!!! Success + !!! Note - In the Turtle view of RDF dataset "MITRE ATT&CK®", you can see the triples inserted by your SPARQL query. + In this query, Jinja replace $outputProperties.uri("graph") and $inputProperties.uri("graph") according to our workflow so the final code executed of this query is, for example: - ```turtle - - owl:imports ; - owl:imports ; - owl:imports - . - ``` + ```sparql + PREFIX owl: + + INSERT DATA { + GRAPH { + + owl:imports . + } + } + ``` + + !!! Success + + In the Turtle view of RDF dataset "MITRE ATT&CK®", you can see the triples inserted by your SPARQL query. -1. In the same workflow add one SPARQL task for each RDF datasets and in output add the RDF dataset "MITRE ATT&CK®". Execute it and save it. + ```turtle + + owl:imports ; + owl:imports ; + owl:imports + . + ``` + +3. In the same workflow add one SPARQL task for each RDF datasets and in output add the RDF dataset "MITRE ATT&CK®". Execute it and save it. ![](23-1-sparql-task.gif) -!!! Success + !!! Success - ![](23-1-workflow-import.png) + ![](23-1-workflow-import.png) - In the Turtle view of RDF dataset "MITRE ATT&CK®", you can see the triples inserted by your SPARQL query. + In the Turtle view of RDF dataset "MITRE ATT&CK®", you can see the triples inserted by your SPARQL query. - ```turtle - - owl:imports ; - owl:imports ; - owl:imports - . - ``` + ```turtle + + owl:imports ; + owl:imports ; + owl:imports + . + ``` ### Test your final SPARQL query @@ -563,67 +562,67 @@ Here, we are creating a new SPARQL Update task to calculate and insert automatic 1. In the same workflow, insert a new SPARQL Update task with this query to calculate the statistics: -* label: Calculate VoID + * label: Calculate VoID -```sparql -PREFIX rdfs: -PREFIX dcterms: -prefix void: - -INSERT -{ - GRAPH $outputProperties.uri("graph") { - $outputProperties.uri("graph") a void:Dataset; - rdfs:label "MITRE ATT&CK®"; - rdfs:comment "MITRE ATT&CK® is a globally-accessible knowledge base of adversary tactics and techniques based on real-world observations."; - void:triples ?triples ; - void:entities ?entities . - } -} -USING $outputProperties.uri("graph") -WHERE { + ```sparql + PREFIX rdfs: + PREFIX dcterms: + prefix void: + + INSERT { - SELECT (COUNT(DISTINCT ?resource) as ?entities) - WHERE { - ?resource a ?class . + GRAPH $outputProperties.uri("graph") { + $outputProperties.uri("graph") a void:Dataset; + rdfs:label "MITRE ATT&CK®"; + rdfs:comment "MITRE ATT&CK® is a globally-accessible knowledge base of adversary tactics and techniques based on real-world observations."; + void:triples ?triples ; + void:entities ?entities . } } - { - SELECT (COUNT(?s) as ?triples) - WHERE { - ?s ?p ?o . + USING $outputProperties.uri("graph") + WHERE { + { + SELECT (COUNT(DISTINCT ?resource) as ?entities) + WHERE { + ?resource a ?class . + } + } + { + SELECT (COUNT(?s) as ?triples) + WHERE { + ?s ?p ?o . + } } } -} -``` + ``` -!!! Tip + !!! Tip - This query uses the variable `$outputProperties.uri("graph")` (Jinja template). If the name of graph changes, the code of the query stays stable in your workflow. + This query uses the variable `$outputProperties.uri("graph")` (Jinja template). If the name of graph changes, the code of the query stays stable in your workflow. -![](23-1-sparql-void.gif) + ![](23-1-sparql-void.gif) -!!! Success + !!! Success - The final triples in the graph `https://attack.mitre.org`after this worflow. + The final triples in the graph `https://attack.mitre.org`after this worflow. - ```turtle - prefix owl: - prefix rdf: - prefix rdfs: - prefix xsd: - - - rdf:type ; - rdfs:comment "MITRE ATT&CK® is a globally-accessible knowledge base of adversary tactics and techniques based on real-world observations." ; - rdfs:label "MITRE ATT&CK®" ; - 28081 ; - 150120 ; - owl:imports - , - , - . - ``` + ```turtle + prefix owl: + prefix rdf: + prefix rdfs: + prefix xsd: + + + rdf:type ; + rdfs:comment "MITRE ATT&CK® is a globally-accessible knowledge base of adversary tactics and techniques based on real-world observations." ; + rdfs:label "MITRE ATT&CK®" ; + 28081 ; + 150120 ; + owl:imports + , + , + . + ``` ### Refresh all automatically @@ -633,76 +632,76 @@ The datasets of Mitre are updated regularly. You may want to update them automat 2. Open your config file: -```bash -cmemc config edit -``` + ```bash + cmemc config edit + ``` 3. Insert your sandbox in your CMEMC config, example with a password grant type: -```bash -[johndo.eccenca.my] -CMEM_BASE_URI=https://johndo.eccenca.my/ -OAUTH_GRANT_TYPE=password -OAUTH_CLIENT_ID=cmemc -OAUTH_USER=johndo@example.com -OAUTH_PASSWORD=XXXXXXXXX -``` + ```bash + [johndo.eccenca.my] + CMEM_BASE_URI=https://johndo.eccenca.my/ + OAUTH_GRANT_TYPE=password + OAUTH_CLIENT_ID=cmemc + OAUTH_USER=johndo@example.com + OAUTH_PASSWORD=XXXXXXXXX + ``` -You need to replace "johndo" by other thing, "" by your login (email) in the sandbox and XXXXXXXXX by your password. Save the file (with VI, :wq). + You need to replace "johndo" by other thing, "" by your login (email) in the sandbox and XXXXXXXXX by your password. Save the file (with VI, :wq). -!!! Tip + !!! Tip - Immediatly, in the file ~/.bashrc, you can specify your sandbox like your instance by default for CMEMC with this line: + Immediatly, in the file ~/.bashrc, you can specify your sandbox like your instance by default for CMEMC with this line: - ```bash - export CMEMC_CONNECTION=johndo.eccenca.my - ``` + ```bash + export CMEMC_CONNECTION=johndo.eccenca.my + ``` -Test: + Test: -```bash -cmemc graph list -# or cmemc -c johndo.eccenca.my graph list -``` + ```bash + cmemc graph list + # or cmemc -c johndo.eccenca.my graph list + ``` -If you can connect it, you can see your knowledge graph "" in the list. + If you can connect it, you can see your knowledge graph "" in the list. -4. You need to know the IDs of your JSON datasets IDs and your workflow ID to implement the command lines with the tool [Corporate Memory Console]() ( +4. You need to know the IDs of your JSON datasets IDs and your workflow ID to implement the command lines with `cmemc`. -![](23-1-collect_IDs.gif) + ![](23-1-collect_IDs.gif) -For example in my demo the JSON datasets and the workflow have these IDs: + For example in my demo the JSON datasets and the workflow have these IDs: -``` -MITREATTCK_3dc114458dfd4c57:MAEntrepriseJSON_14f0f94ed5de5daa -MITREATTCK_3dc114458dfd4c57:MAICSJSON_e024c6433ed523e1 -MITREATTCK_3dc114458dfd4c57:MAMobileJSON_3f890442dad17750 + ```txt + MITREATTCK_3dc114458dfd4c57:MAEntrepriseJSON_14f0f94ed5de5daa + MITREATTCK_3dc114458dfd4c57:MAICSJSON_e024c6433ed523e1 + MITREATTCK_3dc114458dfd4c57:MAMobileJSON_3f890442dad17750 -MITREATTCK_3dc114458dfd4c57:MITREATTCKworkflow_0b8fa5454ef21a00 -``` + MITREATTCK_3dc114458dfd4c57:MITREATTCKworkflow_0b8fa5454ef21a00 + ``` 5. You can now import the file directly of Mitre repository on GitHub and import the files in the sandbox and execute your workflow. -```bash -wget https://raw.githubusercontent.com/mitre-attack/attack-stix-data/master/enterprise-attack/enterprise-attack.json -wget https://raw.githubusercontent.com/mitre-attack/attack-stix-data/master/mobile-attack/mobile-attack.json -wget https://raw.githubusercontent.com/mitre-attack/attack-stix-data/master/ics-attack/ics-attack.json - -cmemc dataset download --replace MITREATTCK_3dc114458dfd4c57:MAEntrepriseJSON_14f0f94ed5de5daa enterprise-attack.json -cmemc dataset download --replace MITREATTCK_3dc114458dfd4c57:MAMobileJSON_3f890442dad17750 mobile-attack.json -cmemc dataset download --replace MITREATTCK_3dc114458dfd4c57:MAICSJSON_e024c6433ed523e1 ics-attack.json -cmemc workflow execute --wait MITREATTCK_3dc114458dfd4c57:MITREATTCKworkflow_0b8fa5454ef21a00 -``` + ```bash + wget https://raw.githubusercontent.com/mitre-attack/attack-stix-data/master/enterprise-attack/enterprise-attack.json + wget https://raw.githubusercontent.com/mitre-attack/attack-stix-data/master/mobile-attack/mobile-attack.json + wget https://raw.githubusercontent.com/mitre-attack/attack-stix-data/master/ics-attack/ics-attack.json + + cmemc dataset download --replace MITREATTCK_3dc114458dfd4c57:MAEntrepriseJSON_14f0f94ed5de5daa enterprise-attack.json + cmemc dataset download --replace MITREATTCK_3dc114458dfd4c57:MAMobileJSON_3f890442dad17750 mobile-attack.json + cmemc dataset download --replace MITREATTCK_3dc114458dfd4c57:MAICSJSON_e024c6433ed523e1 ics-attack.json + cmemc workflow execute --wait MITREATTCK_3dc114458dfd4c57:MITREATTCKworkflow_0b8fa5454ef21a00 + ``` -!!! Success + !!! Success - You can see the result in the shell but also via the "Activities Board". It's useful to follow the errors of your workflows, if you execute a script via a Linux Cron, for example. + You can see the result in the shell but also via the "Activities Board". It's useful to follow the errors of your workflows, if you execute a script via a Linux Cron, for example. - ![](23-1-success-cmemc-activity.png) + ![](23-1-success-cmemc-activity.png) -!!! Tip + !!! Tip - With these command lines, you can now start a cron every day to check the Mitre updates and start refreshing your datasets. + With these command lines, you can now start a cron every day to check the Mitre updates and start refreshing your datasets. ## Exercices @@ -714,53 +713,53 @@ After this tutorial, you want probably to navigate in your new knowledge graph b 2. Create a new SPARQL Update task "convert STIX relationships to rdf statements" with this code: -```sparql -PREFIX ctia: + ```sparql + PREFIX ctia: -INSERT { - GRAPH $outputProperties.uri("graph") { - ?sourceIRI ?propertyIRI ?targetIRI . - } -} -WHERE { - GRAPH $inputProperties.uri("graph") { - ?relationship - ctia:type ctia:relationship ; - ctia:source_ref ?source ; - ctia:target_ref ?target ; - ctia:relationship_type ?property . + INSERT { + GRAPH $outputProperties.uri("graph") { + ?sourceIRI ?propertyIRI ?targetIRI . + } } + WHERE { + GRAPH $inputProperties.uri("graph") { + ?relationship + ctia:type ctia:relationship ; + ctia:source_ref ?source ; + ctia:target_ref ?target ; + ctia:relationship_type ?property . + } - BIND (IRI(CONCAT("https://github.com/mitre-attack/attack-stix-data#",STR(?source))) as ?sourceIRI) - BIND (IRI(CONCAT("https://github.com/mitre/cti/blob/master/USAGE.md#",STR(?property))) as ?propertyIRI) - BIND (IRI(CONCAT("https://github.com/mitre-attack/attack-stix-data#",STR(?target))) as ?targetIRI) -} -``` + BIND (IRI(CONCAT("https://github.com/mitre-attack/attack-stix-data#",STR(?source))) as ?sourceIRI) + BIND (IRI(CONCAT("https://github.com/mitre/cti/blob/master/USAGE.md#",STR(?property))) as ?propertyIRI) + BIND (IRI(CONCAT("https://github.com/mitre-attack/attack-stix-data#",STR(?target))) as ?targetIRI) + } + ``` -This SPARQL query create explicitly the STIX links in the knowledge graph. Here, we create a new inference via a simple query. + This SPARQL query create explicitly the STIX links in the knowledge graph. Here, we create a new inference via a simple query. 3. Create a new Knowledge graph dataset "STIX inferences" with this IRI: -!!! Tip + !!! Tip - Separate always the facts extracted of raw data and the inferences calculate with other graphs. So, you can recalculate your inferences without rebuild all knowledge graph. + Separate always the facts extracted of raw data and the inferences calculate with other graphs. So, you can recalculate your inferences without rebuild all knowledge graph. 4. Split the workflow in two workflows: - * "Transform all STIX data to RDF" to calculate the inferences after RDF triples - ![](23-1-ex-workflow-STIX.png) + * "Transform all STIX data to RDF" to calculate the inferences after RDF triples + ![](23-1-ex-workflow-STIX.png) - * "Assemble the global knowledge graph", it will import all the graphs of projects - ![](23-1-ex-workflow-gen.png) + * "Assemble the global knowledge graph", it will import all the graphs of projects + ![](23-1-ex-workflow-gen.png) 5. Create a new workflow "MITRE ATT&CK® workflow" where you will insert the other workflows, like that: ![](23-1-ex-workflow-global.png) -!!! Success + !!! Success - You can now navigate in your first knowledge graph: - ![](23-1-ex-graph-navigation.gif) + You can now navigate in your first knowledge graph: + ![](23-1-ex-graph-navigation.gif) ### Reconcile automatically the STIX concepts via the Linking tasks @@ -774,8 +773,8 @@ This SPARQL query create explicitly the STIX links in the knowledge graph. Here, The Common Attack Pattern Enumeration and Classification (CAPEC™) effort provides a publicly available catalog of common attack patterns that helps users understand how adversaries exploit weaknesses in applications and other cyber-enabled capabilities. -* Dataset: [https://github.com/mitre/cti/blob/master/capec/2.1/stix-capec.json](https://github.com/mitre/cti/blob/master/capec/2.1/stix-capec.json) -* The CAPEC "ontology": [https://github.com/mitre/cti/blob/master/USAGE-CAPEC.md](https://github.com/mitre/cti/blob/master/USAGE-CAPEC.md) +* Dataset: [https://github.com/mitre/cti/blob/master/capec/2.1/stix-capec.json](https://github.com/mitre/cti/blob/master/capec/2.1/stix-capec.json) +* The CAPEC "ontology": [https://github.com/mitre/cti/blob/master/USAGE-CAPEC.md](https://github.com/mitre/cti/blob/master/USAGE-CAPEC.md) 1. Import the CAPEC dataset in Corporate Memory 2. Create the named graph of CAPEC @@ -788,8 +787,8 @@ STIX uses JSON syntax and can therefore be converted to RDF via Corporate Memory ## Ressources -* [RDF schemas (Model, pattern, etc)](RDF_model_and_pattern.drawio) -* [Archive of CMEM project](MITREATTCK_tutorial.zip) +* [RDF schemas (Model, pattern, etc)](RDF_model_and_pattern.drawio) +* [Archive of CMEM project](MITREATTCK_tutorial.zip) --- diff --git a/docs/build/tutorial-how-to-link-ids-to-osint/lift-data-from-YAML-data-of-hayabusa-sigma/index.md b/docs/build/tutorial-how-to-link-ids-to-osint/lift-data-from-YAML-data-of-hayabusa-sigma/index.md index 7aa2e66e..ec1c1fa1 100644 --- a/docs/build/tutorial-how-to-link-ids-to-osint/lift-data-from-YAML-data-of-hayabusa-sigma/index.md +++ b/docs/build/tutorial-how-to-link-ids-to-osint/lift-data-from-YAML-data-of-hayabusa-sigma/index.md @@ -8,8 +8,8 @@ There are rules for Host-based intrusion detection systems (HIDS) with Hayabusa/ Here, we are working with the Hayabusa/Sigma rules available via GitHub: -- [https://github.com/Yamato-Security/hayabusa-rules](https://github.com/Yamato-Security/hayabusa-rules) -- [https://github.com/SigmaHQ/sigma](https://github.com/Yamato-Security/hayabusa-rules) +- [https://github.com/Yamato-Security/hayabusa-rules](https://github.com/Yamato-Security/hayabusa-rules) +- [https://github.com/SigmaHQ/sigma](https://github.com/Yamato-Security/hayabusa-rules) The problem of interoperability, here, is the YAML format of files, their random position in their folders in their Github projets. Moreover, the same rule can exist in different projects but in this tutorial, we will not fix this problem and we consider the IRI rule is their Web address. In Corporate Memory, we would fix that with the Linked Tool, we will study this tool in a next part of this tutorial. @@ -111,61 +111,61 @@ This new transformer are building the following RDF model for your use case: 5. Create the transformer for "SIGMA Hayabusa rule" to build this RDF model. -Rule object: + Rule object: -- type: `ctis:Rule` + - type: `ctis:Rule` -- IRI: concatenation of "" with the result of this regular expression `^.*?([^\/]*)$` on the rule path + - IRI: concatenation of "" with the result of this regular expression `^.*?([^\/]*)$` on the rule path -![](23-1-iri-rule.png) + ![](23-1-iri-rule.png) -- property `ctis:filename` with the result of this regular expression `^.*?([^\/]*)$` on the value path `rulePath` -- property `rdfs:label` with the value path `title` -- property `rdfs:comment` with the value path `description` -- property `rdfs:seeAlso` with the value path `references` -- property `ctis:mitreAttackTechniqueId` is building with this formula with the value path `tags` - - Filter by regex: `^attack\.t\d+$` - - Regex replace `attack\.t` by `T` + - property `ctis:filename` with the result of this regular expression `^.*?([^\/]*)$` on the value path `rulePath` + - property `rdfs:label` with the value path `title` + - property `rdfs:comment` with the value path `description` + - property `rdfs:seeAlso` with the value path `references` + - property `ctis:mitreAttackTechniqueId` is building with this formula with the value path `tags` + - Filter by regex: `^attack\.t\d+$` + - Regex replace `attack\.t` by `T` -![](23-1-formula-mitreid.png) + ![](23-1-formula-mitreid.png) -- property `rdfs:isDefinedBy` on the value path `rulePath` is building with this formula to link the rules to their Web addresses. - - Add two "Regex replace" - - replace `\./hayabusa-rules/` by `https://github.com/Yamato-Security/hayabusa-rules/blob/main/` - - replace `\./sigma/` by `https://github.com/SigmaHQ/sigma/blob/master/` + - property `rdfs:isDefinedBy` on the value path `rulePath` is building with this formula to link the rules to their Web addresses. + - Add two "Regex replace" + - replace `\./hayabusa-rules/` by `https://github.com/Yamato-Security/hayabusa-rules/blob/main/` + - replace `\./sigma/` by `https://github.com/SigmaHQ/sigma/blob/master/` -![](23-1-rules-isdefinedby.png) + ![](23-1-rules-isdefinedby.png) -So the rulepath `./sigma/rules/windows/process_creation/proc_creation_win_bcdedit_boot_conf_tamper.yml` becomes the link `https://github.com/SigmaHQ/sigma/blob/master/rules/windows/process_creation/proc_creation_win_bcdedit_boot_conf_tamper.yml` and `./hayabusa-rules/hayabusa/sysmon/Sysmon_15_Info_ADS-Created.yml`becomes `https://github.com/Yamato-Security/hayabusa-rules/blob/main/hayabusa/sysmon/Sysmon_11_Med_FileCreated_RuleAlert.yml` + So the rulepath `./sigma/rules/windows/process_creation/proc_creation_win_bcdedit_boot_conf_tamper.yml` becomes the link `https://github.com/SigmaHQ/sigma/blob/master/rules/windows/process_creation/proc_creation_win_bcdedit_boot_conf_tamper.yml` and `./hayabusa-rules/hayabusa/sysmon/Sysmon_15_Info_ADS-Created.yml`becomes `https://github.com/Yamato-Security/hayabusa-rules/blob/main/hayabusa/sysmon/Sysmon_11_Med_FileCreated_RuleAlert.yml` -!!! Tips + !!! Tips - To test your transformer, you can use the tab "Transform execution". Here, the knowledge graph will not be cleared after each workflow or execution to test your transformer because the option "clear graph before workflow" is disabled. However during the steps to build this transformer, you can enable tempory this option to see and test the final transformer. - You need only to disable this option when your transformer is finished. + To test your transformer, you can use the tab "Transform execution". Here, the knowledge graph will not be cleared after each workflow or execution to test your transformer because the option "clear graph before workflow" is disabled. However during the steps to build this transformer, you can enable tempory this option to see and test the final transformer. + You need only to disable this option when your transformer is finished. -!!! Success + !!! Success - Your example of rule exists now in your knowledge graph: - ![](23-1-success-extract-rule2.png) - ![](23-1-success-extract-rule.png) + Your example of rule exists now in your knowledge graph: + ![](23-1-success-extract-rule2.png) + ![](23-1-success-extract-rule.png) 6. Make the workflow "Import rules" with one input ![](23-1-success-workflow.png) -And don't forget to allow the replacement of JSON dataset because it allows to replace this specific JSON by all other rules during the execution of this worflow. + And don't forget to allow the replacement of JSON dataset because it allows to replace this specific JSON by all other rules during the execution of this worflow. -![](23-1-workflow-allow-replacement.png) + ![](23-1-workflow-allow-replacement.png) -![](23-1-add-worflow.gif) + ![](23-1-add-worflow.gif) 7. Copy the workflow ID ![](23-1-id-worflow.gif) -!!! Success + !!! Success - In this example the ID of workflow is `RulesHayabusaSigma_671e1f43d94bbc36:Importrules_6ccbc14b656c75c9` + In this example the ID of workflow is `RulesHayabusaSigma_671e1f43d94bbc36:Importrules_6ccbc14b656c75c9` ## Apply the worflow to all files @@ -232,9 +232,9 @@ Here, we learnt how to generate a knowledge graph with files in input with Corpo ## Ressources -- [RDF schemas (Model, pattern, etc)](RDF_model_and_pattern.drawio) -- [script 1](importRules.sh) -- [script 2](importRules2.sh) +- [RDF schemas (Model, pattern, etc)](RDF_model_and_pattern.drawio) +- [script 1](importRules.sh) +- [script 2](importRules2.sh) --- diff --git a/docs/build/tutorial-how-to-link-ids-to-osint/link-IDS-event-to-KG-via-cmem/index.md b/docs/build/tutorial-how-to-link-ids-to-osint/link-IDS-event-to-KG-via-cmem/index.md index 84b024c6..cc9e4762 100644 --- a/docs/build/tutorial-how-to-link-ids-to-osint/link-IDS-event-to-KG-via-cmem/index.md +++ b/docs/build/tutorial-how-to-link-ids-to-osint/link-IDS-event-to-KG-via-cmem/index.md @@ -6,9 +6,10 @@ In this tutorial, we are using the Splunk app "Investigate lateral movements wit In the demo of this Splunk app via the video 1, the user selects the data about one investigation via Splunk and generate a bash script to export these data via the Splunk API in tempory graphs in Corporate Memory for each investigation. -![](splunk-app-demo-poc-app.gif) - -*Video 1: Splunk dashboards of the Splunk app "Investigate lateral movements with a knowledge graph"* +
+![Video 1: Splunk dashboards of the Splunk app "Investigate lateral movements with a knowledge graph"](splunk-app-demo-poc-app.gif) +
Video 1: Splunk dashboards of the Splunk app "Investigate lateral movements with a knowledge graph"
+
In this page, we are showing how, we : @@ -27,7 +28,7 @@ This app is not directly connected to your Corporate Memory instance. The custom Position of these scripts in the folders of this app: -``` +```shell +---bin | Investigation.py +---cmem @@ -41,43 +42,49 @@ Moreover, a settings file is necessary to insert the credentials of Splunk (like In this Proof of Concept, we have implemented two types of investigation: -- high-level with the data of alerts of Zeek and Hayabusa/Sigma -- low-level with the data of Suricata and Sysmon +- high-level with the data of alerts of Zeek and Hayabusa/Sigma +- low-level with the data of Suricata and Sysmon For each investigation, an analyst selects and navigates in the data with two dashboards for each type of investigation: -- one dashboard (see figure 1) to select the data to transfer to knowledge graph: typeA_request.xml -- one dashboard (see figure 2,3,4) to navigate in the knowledge graph: typeA_dashboard.xml - -![](poc-app_request.png) - -*Figure 1: Splunk dashboard to select the data before executing a high-level investigation* +- one dashboard (see figure 1) to select the data to transfer to knowledge graph: typeA_request.xml +- one dashboard (see figure 2,3,4) to navigate in the knowledge graph: typeA_dashboard.xml -![](poc-app_high_level_computers.png) +
+![Figure 1: Splunk dashboard to select the data before executing a high-level investigation](poc-app_request.png) +
Figure 1: Splunk dashboard to select the data before executing a high-level investigation
+
-*Figure 2: High-level investigation dashboard with the list of computers implicated in the incident* +
+![Figure 2: High-level investigation dashboard with the list of computers implicated in the incident](poc-app_high_level_computers.png) +
Figure 2: High-level investigation dashboard with the list of computers implicated in the incident
+
-![](poc-app_high_level_period.png) +
+![Figure 3: High-level investigation dashboard with the panel to select a specific period during an incident according to IoCs details](poc-app_high_level_period.png) +
Figure 3: High-level investigation dashboard with the panel to select a specific period during an incident according to IoCs details
+
-*Figure 3: High-level investigation dashboard with the panel to select a specific period during an incident according to IoCs details* - -![](poc-app_low_level.png) - -*Figure 4: low-level investigation dashboard contains the command lines of Windows processus rised Suricata alerts during the period selected by the analyst in a high-level investigation dashboard.* +
+![Figure 4: low-level investigation dashboard contains the command lines of Windows processus rised Suricata alerts during the period selected by the analyst in a high-level investigation dashboard](poc-app_low_level.png) +
Figure 4: low-level investigation dashboard contains the command lines of Windows processus rised Suricata alerts during the period selected by the analyst in a high-level investigation dashboard
+
The idea is the analyst can do an investigation high-level without using a lot of ressources (a little graph) but when he want to see the suspicious processus on one computer in a specific period, he can ask an investigation low-level with a maximum of details. To follow the calculation of investigations and free memory when one investigation is closed, we developed another dashboard "investigation_list.xml" (see figure 5). This dashboard prints the status of investigations actually in the knowledge graph. Here, each investigation is saved in tempory graphs and the analyst can create and delete them directly in Splunk. -![](poc-app_list_investigations.png) - -*Figure 5: The dashboard "investigation list" shows all the tempory graph actually in the knowledge graph. The analyst can open an investigation, see the SPL query generated when he has created an investigation and delete it when he want.* +
+![Figure 5: The dashboard "investigation list" shows all the tempory graph actually in the knowledge graph. The analyst can open an investigation, see the SPL query generated when he has created an investigation and delete it when he want.](poc-app_list_investigations.png) +
Figure 5: The dashboard "investigation list" shows all the tempory graph actually in the knowledge graph. The analyst can open an investigation, see the SPL query generated when he has created an investigation and delete it when he want.
+
With these interfaces to manage and calculate different investigations with different levels of details, we imagined a first method to "follow lateral movements" (see figure 6) in order to understand the objectives of the incident. We hope this PoC will "Accelerate Cyber Threat Hunting". -![](poc-app_hunt.png) - -*Figure 6: Analyst can select a computer and a period to analyze the suspicious processus implicated in Suricata alerts. So, an analyst can follow the "lateral movements" and see the command lines executed by these suspicious processus.* +
+![Figure 6: Analyst can select a computer and a period to analyze the suspicious processus implicated in Suricata alerts. So, an analyst can follow the "lateral movements" and see the command lines executed by these suspicious processus.](poc-app_hunt.png) +
Figure 6: Analyst can select a computer and a period to analyze the suspicious processus implicated in Suricata alerts. So, an analyst can follow the "lateral movements" and see the command lines executed by these suspicious processus.
+
## Manage the graphs of your application @@ -97,9 +104,9 @@ This manner to manage the graphs has been applied in this app, so an analyst is This token contains a json object where: -- "index" the list of splunk indexes of the SPL query -- "search" the second part of the SPL query -- "workflowID" the ID of workflow in Corporate Memory to convert the raw data of Splunk in RDF in a tempory graph +- "index" the list of splunk indexes of the SPL query +- "search" the second part of the SPL query +- "workflowID" the ID of workflow in Corporate Memory to convert the raw data of Splunk in RDF in a tempory graph An analyst can import as many sources as needed with several tokens, ie. "source_1", "source_2", etc. @@ -132,8 +139,8 @@ There are not consensus about the manner to calculate inferences on the RDF data We use two manners to calculate new inferences: -- when the inference is simple to calculate with SPARQL, we use a SPARQL update query with parameters in Corporate Memory (like "inferenceID" to build the name of destination tempory graph). "queryIRI" is the IRI of the query in the catalog of Corporate Memory. -- when the inference is complex to calculate, we use a workflow of Corporate Memory. +- when the inference is simple to calculate with SPARQL, we use a SPARQL update query with parameters in Corporate Memory (like "inferenceID" to build the name of destination tempory graph). "queryIRI" is the IRI of the query in the catalog of Corporate Memory. +- when the inference is complex to calculate, we use a workflow of Corporate Memory. With these tokens "source_1", "source_2", etc and "inferences" in the dashboard, the app can generate a bash script for CMEMC. diff --git a/docs/build/tutorial-how-to-link-ids-to-osint/link-IDS-event-to-KG/index.md b/docs/build/tutorial-how-to-link-ids-to-osint/link-IDS-event-to-KG/index.md index 98d7fabc..776a7418 100644 --- a/docs/build/tutorial-how-to-link-ids-to-osint/link-IDS-event-to-KG/index.md +++ b/docs/build/tutorial-how-to-link-ids-to-osint/link-IDS-event-to-KG/index.md @@ -6,9 +6,10 @@ In this tutorial, we are using the Linked Data App for Splunk. This app contains In the demo of this app in the video 1, the user selects the indexes of his investigation and select an alert message to open its sources on the Web before searching manually via the Splunk interfaces. Splunk, automatically, refreshes the SPARQL queries in the dashboard after each interaction of user. -![](splunk-app-demo-LD-app.gif) - -*Video 1: Splunk dashboards of the Linked Data App* +
+![Video 1: Splunk dashboards of the Linked Data App](splunk-app-demo-LD-app.gif) +
Video 1: Splunk dashboards of the Linked Data App
+
In this tutorial, we learn to: @@ -25,15 +26,17 @@ The "Linked Data App" extends Splunk Search Processing Language (SPL) to support 2. Open the App window in Splunk via the icon "tools" (see figure 1) -![](splunk_apps_menu.png) - -*Figure 1: In the top of the list of installed Splunk apps, you need to click on the icon "tools" to open the window to manage your apps* - -3. Upload the app in Splunk (see video 2) +
+![Figure 1: In the top of the list of installed Splunk apps, you need to click on the icon "tools" to open the window to manage your apps](splunk_apps_menu.png) +
Figure 1: In the top of the list of installed Splunk apps, you need to click on the icon "tools" to open the window to manage your apps
+
-![](splunk-app-install.gif) +1. Upload the app in Splunk (see video 2) -*Video 2: When the tar.gz of the "Linked Data App", you can upload it manually directly in Splunk.* +
+![Video 2: When the tar.gz of the "Linked Data App", you can upload it manually directly in Splunk](splunk-app-install.gif) +
Video 2: When the tar.gz of the "Linked Data App", you can upload it manually directly in Splunk
+
!!! Tip @@ -61,7 +64,7 @@ vi default/settings.conf You have an example of configuration for the eccenca sandbox SPARQL endpoint in the file `default/settings_template_sandbox.conf` (and another example via Oauth2 secret ID in the file `default/settings_template_oauth_secret_id.conf`). -2. Insert your credentials in the the file `settings.conf`, ie. replace `johndo` by the name of your sandbox (endpointRead, token_endpoint), `johndo@example.com` by your email and `XXXXXXXXX` by your password. Don't change the parameters OAUTH_CLIENT_ID and OAUTH_GRANT_TYPE. +1. Insert your credentials in the the file `settings.conf`, ie. replace `johndo` by the name of your sandbox (endpointRead, token_endpoint), `johndo@example.com` by your email and `XXXXXXXXX` by your password. Don't change the parameters OAUTH_CLIENT_ID and OAUTH_GRANT_TYPE. ```ini [config:default] @@ -78,11 +81,11 @@ OAUTH_USER=johndo@example.com OAUTH_PASSWORD=XXXXXXXXX ``` -3. Restart after your Splunk instance (via the administration windows) +1. Restart after your Splunk instance (via the administration windows) -4. Test your sandbox endpoint in Splunk with this SPL query: +2. Test your sandbox endpoint in Splunk with this SPL query: -``` +```spl | sparql query=" select * @@ -104,7 +107,7 @@ endpointRead=https://query.wikidata.org/sparql Restart after your Splunk instance and request in Splunk your endpoint with the parameter config (here wikidata) to select the config to use in the file `settings.conf`: -``` +```spl | sparql config="wikidata" query=" @@ -124,9 +127,10 @@ Restart after your Splunk instance and request in Splunk your endpoint with the To work, our example of dashboard need to have Splunk indexes of IoCs. We cannot share our indexes but you can modify our example with your own SPL queries according to your Splunk indexes. -![](demo_ld.png) - -*Figure 2: Dashboard with SPARQL commands and the script `table_html.js` to print the HTML and to open Web pages of alerts' references* +
+![Figure 2: Dashboard with SPARQL commands and the script `table_html.js` to print the HTML and to open Web pages of alerts' references](demo_ld.png) +
Figure 2: Dashboard with SPARQL commands and the script `table_html.js` to print the HTML and to open Web pages of alerts' references
+
!!! Tip diff --git a/docs/build/variables/index.md b/docs/build/variables/index.md index 0aafd594..7ffeca6c 100644 --- a/docs/build/variables/index.md +++ b/docs/build/variables/index.md @@ -13,17 +13,17 @@ These variables define various aspects of the integration tasks, such as the sou The variables are not technically typed. They can be used in most Build configuration and input fields that take inputs of the following data types: -- simple text/string parameters (any string), -- integer parameters (any integer), -- and boolean values (`true`/`false`). +- simple text/string parameters (any string), +- integer parameters (any integer), +- and boolean values (`true`/`false`). Two kinds of variables can be defined: -**Global variables** +`Global variables` : It is defined by the administrator in the configuration file at deployment time and cannot be set by a normal user. -**Project variables (User-defined)** +`Project variables (User-defined)` : It is defined by the user in the UI. Project variables can only be used in the same project. @@ -189,4 +189,3 @@ Click on the symbol **{#}** it turns blue in color. It means the variable's feat In order to allow the automation of activities with build variables from external processes, the Corporate Memory command line interfaces cmemc has a dedicated [`project variable` command group](../../automate/cmemc-command-line-interface/command-reference/project/variable/index.md) for this. Please have a look at command group documentation to learn how to use these commands. - diff --git a/docs/build/workflow-reconfiguration/index.md b/docs/build/workflow-reconfiguration/index.md index 80e300ae..1321f61e 100644 --- a/docs/build/workflow-reconfiguration/index.md +++ b/docs/build/workflow-reconfiguration/index.md @@ -57,4 +57,3 @@ After this is done, you can reconfigure any workflow operator that uses this par Tutorials that showcase this function in an application context: - [Loading JDBC datasets incrementally](../loading-jdbc-datasets-incrementally/index.md) - diff --git a/docs/consume/consume-graphs-in-apache-kafka/index.md b/docs/consume/consume-graphs-in-apache-kafka/index.md index 69b54dd9..a0e6d165 100644 --- a/docs/consume/consume-graphs-in-apache-kafka/index.md +++ b/docs/consume/consume-graphs-in-apache-kafka/index.md @@ -41,4 +41,3 @@ Once you installed the package, you can use the Kafka Producer by simply creatin ![Create new Item and search for `kafka`](create-new-item-kafka.png "Create new Item and search for `kafka`") Follow the in-app documentation on how to configure the task (e.g. for providing credentials or preparing data to be sent in messages). - diff --git a/docs/consume/consuming-graphs-in-power-bi/index.md b/docs/consume/consuming-graphs-in-power-bi/index.md index d21d2c33..68f2d7a8 100644 --- a/docs/consume/consuming-graphs-in-power-bi/index.md +++ b/docs/consume/consuming-graphs-in-power-bi/index.md @@ -18,14 +18,14 @@ The latest (unsigned) version of our Power-BI-Connector is available from its so - [eccenca github.com repository](https://github.com/eccenca/power-bi-connector/tags) (unsigned .mez file)![release](https://img.shields.io/github/release-date/eccenca/power-bi-connector?style=plastic) ![tag](https://img.shields.io/github/v/tag/eccenca/power-bi-connector?style=plastic) - [eccenca Corporate Memory Releases](https://releases.eccenca.com/power-bi-connector/) (signed .pqx file) - - Thumbprint of the signature: **FB6C562BD0B08107AAA420EDDE94507420C7FE1A** + - Thumbprint of the signature: **FB6C562BD0B08107AAA420EDDE94507420C7FE1A** ## Installation - Download the `.pqx` or `.mez` file from the locations linked above. - Move the file into the folder `Documents\Power BI Desktop\Custom Connectors` . - - Create the folder if it does not exist. - - In case you are running Windows on Parallels Desktop: Do not use the Local `Disk\Users\UserName\Documents` folder but your shared folder with macOS. + - Create the folder if it does not exist. + - In case you are running Windows on Parallels Desktop: Do not use the Local `Disk\Users\UserName\Documents` folder but your shared folder with macOS. - Register the Thumbprint (for .pqx) or setup PowerBI Desktop to allow any 3rd party connector (for .pqx or .mez) *(we recommend to register the Thumbprint)* ??? Setup diff --git a/docs/consume/consuming-graphs-with-redash/index.md b/docs/consume/consuming-graphs-with-redash/index.md index b3aa2387..331f716f 100644 --- a/docs/consume/consuming-graphs-with-redash/index.md +++ b/docs/consume/consuming-graphs-with-redash/index.md @@ -40,4 +40,3 @@ To get familiar with Redash, please have a look at the Redash user guide, especi In order to query eccenca Corporate Memory data sources in Redash, you have to formulate your query with SPARQL: ![Redash: Query with SPARQL](3-edit-query.png "Redash: Query with SPARQL") - diff --git a/docs/consume/index.md b/docs/consume/index.md index b2f5fd70..6e4278ee 100644 --- a/docs/consume/index.md +++ b/docs/consume/index.md @@ -17,44 +17,42 @@ Since not all applications allow the direct use of SPARQL, this section includes **:octicons-people-24: Intended audience**: Linked Data Experts -
-- :other-powerbi: [Power BI](consuming-graphs-in-power-bi/index.md) +- :other-powerbi: [Power BI](consuming-graphs-in-power-bi/index.md) --- Learn how to consume data from your Corporate Memory Knowledge Graph with our Microsoft Power-BI-Connector. -- :material-view-dashboard: [Redash](consuming-graphs-with-redash/index.md) +- :material-view-dashboard: [Redash](consuming-graphs-with-redash/index.md) --- Create Dashboards based on your Knowledge Graphs with the open-source application Redash. -- :eccenca-artefact-dataset-sqlendpoint: [SQL Databases](consuming-graphs-with-sql-databases/index.md) +- :eccenca-artefact-dataset-sqlendpoint: [SQL Databases](consuming-graphs-with-sql-databases/index.md) --- If direct access to the knowledge graph is not sufficient, fragments of the Knowledge Graph may also be pushed into external SQL databases. -- :material-api: [Custom APIs](provide-data-in-any-format-via-a-custom-api/index.md) +- :material-api: [Custom APIs](provide-data-in-any-format-via-a-custom-api/index.md) --- Learn how to provide data via a customized Corporate Memory API in a text format of your choice and how to consume it in your applications. -- :simple-neo4j: [Neo4j](populate-data-to-neo4j/index.md) +- :simple-neo4j: [Neo4j](populate-data-to-neo4j/index.md) --- Learn how to populate graphs to Neo4j. -- :material-apache-kafka: [Apache Kafka](consume-graphs-in-apache-kafka/index.md) +- :material-apache-kafka: [Apache Kafka](consume-graphs-in-apache-kafka/index.md) --- Use a Apache Kafka Producer in order to export parts of your Knowledge Graph as a message stream.
- diff --git a/docs/consume/populate-data-to-neo4j/index.md b/docs/consume/populate-data-to-neo4j/index.md index 8e190ba3..d29c7ae7 100644 --- a/docs/consume/populate-data-to-neo4j/index.md +++ b/docs/consume/populate-data-to-neo4j/index.md @@ -37,9 +37,9 @@ To upload multiple files together as an input: 1. In Corporate Memory, click Projects under **BUILD** in the navigation on the left side of the page. ![Menu > BUILD > Projects](pdtn-menu-projects.png){width="50%"} -2. Click Create at the top of the page.  +2. Click Create at the top of the page. 3. In Create new item window, select Project and click Add. The Create new item of type Project window appears. -4. Fill in the required details such as Title and Description.  Alternatively, import the existing project by clicking Import Project File and selecting the file from your system.  +4. Fill in the required details such as Title and Description.  Alternatively, import the existing project by clicking Import Project File and selecting the file from your system. 5. Click Create. Your project (Northwind) is created. ![Project Northwind](pdtn-project-northwind.png) 6. In your project, click Create Item. @@ -57,7 +57,7 @@ To upload multiple files together as an input: A Neo4j dataset holding a Labeled Property Graph (LPG) representation is one of the outputs of the process. Perform the following steps to create a Neo4j dataset: -1. In your existing project, click Create to create a new item.  +1. In your existing project, click Create to create a new item. 2. In the item category Dataset select Neo4j. ![Create Neo4j dataset](pdtn-create-dataset-neo4j.png) 3. Click Add. @@ -248,7 +248,7 @@ To integrate all the transformations, perform the following steps: 1. Navigate to the project. 2. Click Create -3. In the Create new item window, select Workflow and click Add.  +3. In the Create new item window, select Workflow and click Add. 4. In the Create new item of type Transform window, for this tutorial, enter the following: - Label: _Northwind Workflow_ - Click Create @@ -273,4 +273,3 @@ Optionally, you can use the same transformation and workflow to render the resul ![Add Knowledge Graph output](pdtn-add-kg-dataset-into-workflow.png) 2. The results can then be reviewed in the Knowledge Graph module, e.g., explored visually: ![Results in Corporate Memory](pdtn-ontodia-result-view.png) - diff --git a/docs/consume/provide-data-in-any-format-via-a-custom-api/index.md b/docs/consume/provide-data-in-any-format-via-a-custom-api/index.md index b0209425..4ac61c3d 100644 --- a/docs/consume/provide-data-in-any-format-via-a-custom-api/index.md +++ b/docs/consume/provide-data-in-any-format-via-a-custom-api/index.md @@ -176,4 +176,3 @@ A non-streaming result set (the SPARQL query) is limited to 1000 elements. If more results are expected *Is Streaming* should be set to true. If *Is Streaming* is set to `true` the Jinja Template has to resolve a `result` variable (without the '`s`'), which is a single query result. The template engine iterates over the results, i.e. the Body template is repeated for each query result. - diff --git a/docs/deploy-and-configure/configuration/access-conditions/index.md b/docs/deploy-and-configure/configuration/access-conditions/index.md index 6a6ea4bb..1b1bcc33 100644 --- a/docs/deploy-and-configure/configuration/access-conditions/index.md +++ b/docs/deploy-and-configure/configuration/access-conditions/index.md @@ -10,7 +10,7 @@ tags: Access Conditions specify access rights for users and groups to graphs and actions (1). { .annotate } -1. Graphs identify specific Knowledge Graphs. +1. Graphs identify specific Knowledge Graphs. Actions identify specific parts or components of the platform, such as the query catalog or the data integration system (Build). Access Conditions are managed in a special system graph, so write access to this graph needs to be handled carefully. @@ -196,7 +196,6 @@ This screen is split into two main areas: - Second, the list of all access conditions which contributed to the effective access rights. This section allows to see which access conditions matched the principal and which access rights they grant. - ### Command line interface With [cmemc](../../../automate/cmemc-command-line-interface/index.md) you can use an additional command line based interface to manage access conditions. @@ -209,7 +208,6 @@ The important command groups for managing principals and access conditions are: The following session demonstrates how to create a new user, set a password and grant access to certain areas. - ``` shell-session $ cmemc admin acl list No access conditions found. Use the `admin acl create` command to create a new access condition. @@ -318,4 +316,3 @@ The following list, provides grants which work together: - Allowed Actions: **Build - Workspace** (`:Build`) - Allow write graphs: - **All Graphs** (`https://vocab.eccenca.com/auth/AllGraphs`) - diff --git a/docs/deploy-and-configure/configuration/caveats/index.md b/docs/deploy-and-configure/configuration/caveats/index.md index 3656a834..322f2b9d 100644 --- a/docs/deploy-and-configure/configuration/caveats/index.md +++ b/docs/deploy-and-configure/configuration/caveats/index.md @@ -63,6 +63,6 @@ For Explore backend (DataPlatform) you have to set this in `application.yaml` or ### Useful Documentation -- [AWS VPN Documentation](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/networking-connecting-services.html) -- [AWS Network load Balancers](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html) -- [GraphDB with load balancers](https://graphdb.ontotext.com/documentation/10.7/aws-deployment.html#setting-up-the-load-balancer) +- [AWS VPN Documentation](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/networking-connecting-services.html) +- [AWS Network load Balancers](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html) +- [GraphDB with load balancers](https://graphdb.ontotext.com/documentation/10.7/aws-deployment.html#setting-up-the-load-balancer) diff --git a/docs/deploy-and-configure/configuration/dataintegration/activity-reference/index.md b/docs/deploy-and-configure/configuration/dataintegration/activity-reference/index.md index 7ef90a29..6a682a4e 100644 --- a/docs/deploy-and-configure/configuration/dataintegration/activity-reference/index.md +++ b/docs/deploy-and-configure/configuration/dataintegration/activity-reference/index.md @@ -16,17 +16,14 @@ The following activities are available for each project. Generates matches between schema paths and datasets based on the schema discovery and profiling information of the datasets. - | Parameter | Type | Description | Example | | ---------------------- | ------------- | ------------------ | -------------------------- | -| datasetUri | String | If set, run dataset matching only for this particular dataset. | +| datasetUri | String | If set, run dataset matching only for this particular dataset. | | The identifier for this plugin is `DatasetMatcher`. It can be found in the package `com.eccenca.di.datamatching`. - - ## Task Activities The following activities are available for different types of tasks. @@ -42,8 +39,6 @@ The identifier for this plugin is `ExecuteRestTask`. It can be found in the package `com.eccenca.di.workflow.operators.rest`. - - ### Dataset #### Dataset profiler @@ -52,20 +47,18 @@ Generates profiling data of a dataset, e.g. data types, statistics etc. | Parameter | Type | Description | Example | | ---------------------- | ------------- | ------------------ | -------------------------- | -| datasetUri | String | Optional URI of the dataset resource that should be profiled. If not specified an URI will be generated. | -| uriPrefix | String | Optional URI prefix that is prepended to every generated URI, e.g. property URIs for every schema path. If not specified an URI prefix will be generated. | -| entitySampleLimit | String | How many entities should be sampled for the profiling. If set to zero or a negative value, all entities will be considered. If left blank the configured default value is used. | -| timeLimit | String | The time in milliseconds that each of the schema extraction step and profiling step should spend on. Leave blank for unlimited time. | -| classProfilingLimit | int | The maximum number of classes that are profiled from the extracted schema. | -| schemaEntityLimit | int | The maximum number of overall schema entities (types, properties/attributes) that will be extracted. | -| executionType | String | The execution type to be used. At the moment, only 'LEGACY' is supported. | +| datasetUri | String | Optional URI of the dataset resource that should be profiled. If not specified an URI will be generated. | | +| uriPrefix | String | Optional URI prefix that is prepended to every generated URI, e.g. property URIs for every schema path. If not specified an URI prefix will be generated. | | +| entitySampleLimit | String | How many entities should be sampled for the profiling. If set to zero or a negative value, all entities will be considered. If left blank the configured default value is used. | | +| timeLimit | String | The time in milliseconds that each of the schema extraction step and profiling step should spend on. Leave blank for unlimited time. | | +| classProfilingLimit | int | The maximum number of classes that are profiled from the extracted schema. | | +| schemaEntityLimit | int | The maximum number of overall schema entities (types, properties/attributes) that will be extracted. | | +| executionType | String | The execution type to be used. At the moment, only 'LEGACY' is supported. | | The identifier for this plugin is `DatasetProfiler`. It can be found in the package `com.eccenca.di.profiling`. - - #### SQL endpoint status Shows the SQL endpoint status. @@ -75,8 +68,6 @@ The identifier for this plugin is `SqlEndpointStatus`. It can be found in the package `com.eccenca.di.sql.spark.endpoint.activity`. - - #### Types cache Holds the most frequent types in a dataset. @@ -86,8 +77,6 @@ The identifier for this plugin is `TypesCache`. It can be found in the package `org.silkframework.workspace.activity.dataset`. - - ### LinkSpecification #### Active learning @@ -96,48 +85,42 @@ Executes an active learning iteration. | Parameter | Type | Description | Example | | ---------------------- | ------------- | ------------------ | -------------------------- | -| fixedRandomSeed | boolean | No description | +| fixedRandomSeed | boolean | No description | | The identifier for this plugin is `ActiveLearning`. It can be found in the package `org.silkframework.learning.active`. - - #### Active learning (find comparison pairs) Suggest comparison pairs for the current linking task. | Parameter | Type | Description | Example | | ---------------------- | ------------- | ------------------ | -------------------------- | -| fixedRandomSeed | boolean | No description | +| fixedRandomSeed | boolean | No description | | The identifier for this plugin is `ActiveLearning-ComparisonPairs`. It can be found in the package `org.silkframework.learning.active.comparisons`. - - #### Evaluate linking Evaluates the linking task by generating links. | Parameter | Type | Description | Example | | ---------------------- | ------------- | ------------------ | -------------------------- | -| includeReferenceLinks | boolean | Do not generate a link for which there is a negative reference link while always generating positive reference links. | -| useFileCache | boolean | Use a file cache. This avoids memory overflows for big files. | -| partitionSize | int | The number of entities in a single partition in the cache. | -| generateLinksWithEntities | boolean | Generate detailed information about the matched entities. If set to false, the generated links won't be shown in the Workbench. | -| writeOutputs | boolean | Write the generated links to the configured output of this task. | -| linkLimit | int | If defined, the execution will stop after the configured number of links is reached.\This is just a hint and the execution may produce slightly fewer or more links. | -| timeout | int | Timeout in seconds after that the matching task of an evaluation should be aborted. Set to 0 or negative to disable the timeout. | +| includeReferenceLinks | boolean | Do not generate a link for which there is a negative reference link while always generating positive reference links. | | +| useFileCache | boolean | Use a file cache. This avoids memory overflows for big files. | | +| partitionSize | int | The number of entities in a single partition in the cache. | | +| generateLinksWithEntities | boolean | Generate detailed information about the matched entities. If set to false, the generated links won't be shown in the Workbench. | | +| writeOutputs | boolean | Write the generated links to the configured output of this task. | | +| linkLimit | int | If defined, the execution will stop after the configured number of links is reached.\This is just a hint and the execution may produce slightly fewer or more links. | | +| timeout | int | Timeout in seconds after that the matching task of an evaluation should be aborted. Set to 0 or negative to disable the timeout. | | The identifier for this plugin is `EvaluateLinking`. It can be found in the package `org.silkframework.workspace.activity.linking`. - - #### Execute linking Executes the linking task using the configured execution. @@ -147,8 +130,6 @@ The identifier for this plugin is `ExecuteLinking`. It can be found in the package `org.silkframework.workspace.activity.linking`. - - #### Linking paths cache Holds the most frequent paths for the selected entities. @@ -158,8 +139,6 @@ The identifier for this plugin is `LinkingPathsCache`. It can be found in the package `org.silkframework.workspace.activity.linking`. - - #### Reference entities cache For each reference link, the reference entities cache holds all values of the linked entities. @@ -169,8 +148,6 @@ The identifier for this plugin is `ReferenceEntitiesCache`. It can be found in the package `org.silkframework.workspace.activity.linking`. - - ### Scheduler #### Activate @@ -182,8 +159,6 @@ The identifier for this plugin is `ExecuteScheduler`. It can be found in the package `com.eccenca.di.scheduler`. - - ### ScriptTask #### Execute Script @@ -195,8 +170,6 @@ The identifier for this plugin is `ExecuteScript`. It can be found in the package `com.eccenca.di.scripting.scala`. - - ### TransformSpecification #### Execute transform @@ -205,14 +178,12 @@ Executes the transformation. | Parameter | Type | Description | Example | | ---------------------- | ------------- | ------------------ | -------------------------- | -| limit | IntOptionParameter | Limits the maximum number of entities that are transformed. | +| limit | IntOptionParameter | Limits the maximum number of entities that are transformed. | | The identifier for this plugin is `ExecuteTransform`. It can be found in the package `org.silkframework.workspace.activity.transform`. - - #### Transform paths cache Holds the most frequent paths for the selected entities. @@ -222,8 +193,6 @@ The identifier for this plugin is `TransformPathsCache`. It can be found in the package `org.silkframework.workspace.activity.transform`. - - #### Target vocabulary cache Holds the target vocabularies @@ -233,8 +202,6 @@ The identifier for this plugin is `VocabularyCache`. It can be found in the package `org.silkframework.workspace.activity.transform`. - - ### Workflow #### Execute locally @@ -246,8 +213,6 @@ The identifier for this plugin is `ExecuteLocalWorkflow`. It can be found in the package `org.silkframework.workspace.activity.workflow`. - - ### WorkflowExecution #### Generate Spark assembly @@ -256,16 +221,14 @@ Generate project and Spark assembly artifacts and deploy them using the specifie | Parameter | Type | Description | Example | | ---------------------- | ------------- | ------------------ | -------------------------- | -| executeStaging | boolean | Execute loading phase | -| executeTransform | boolean | Execute transform phase | -| executeLoading | boolean | Execute staging phase | +| executeStaging | boolean | Execute loading phase | | +| executeTransform | boolean | Execute transform phase | | +| executeLoading | boolean | Execute staging phase | | The identifier for this plugin is `DeploySparkWorkflow`. It can be found in the package `com.eccenca.di.spark`. - - #### Default execution Executes a workflow with the executor defined in the configuration @@ -275,22 +238,18 @@ The identifier for this plugin is `ExecuteDefaultWorkflow`. It can be found in the package `com.eccenca.di.spark`. - - #### Execute operator Executes a workflow on with an executor that uses Apache Spark. Depending on the Spark configuration it can still run on a single local machine or on a cluster. | Parameter | Type | Description | Example | | ---------------------- | ------------- | ------------------ | -------------------------- | -| operator | TaskReference | The workflow to execute. | +| operator | TaskReference | The workflow to execute. | | The identifier for this plugin is `ExecuteSparkOperator`. It can be found in the package `com.eccenca.di.spark`. - - #### Execute on Spark Executes a workflow on with an executor that uses Apache Spark. Depending on the Spark configuration it can still run on a single local machine or on a cluster. @@ -300,36 +259,29 @@ The identifier for this plugin is `ExecuteSparkWorkflow`. It can be found in the package `com.eccenca.di.spark`. - - #### Execute with payload Executes a workflow with custom payload. | Parameter | Type | Description | Example | | ---------------------- | ------------- | ------------------ | -------------------------- | -| configuration | MultilineStringParameter | No description | -| configurationType | String | No description | -| optionalPrimaryResourceManager | PluginObjectParameter | | +| configuration | MultilineStringParameter | No description | | +| configurationType | String | No description | | +| optionalPrimaryResourceManager | PluginObjectParameter | | | The identifier for this plugin is `ExecuteWorkflowWithPayload`. It can be found in the package `org.silkframework.workbench.workflow`. - - #### Generate view Generate and share a view on a workflow executed by the Spark executor. Executes a workflow on Spark and generates a SparkSQL temporary table instead of serializing the result. The table can be accessed via JDBC | Parameter | Type | Description | Example | | ---------------------- | ------------- | ------------------ | -------------------------- | -| caching | boolean | Optional parameter that enables caching (default=false). | -| userDefinedName | String | Optional View name that is used when a view on a non virtual is generated (default = [TASK-ID]_generated_view). | +| caching | boolean | Optional parameter that enables caching (default=false). | | +| userDefinedName | String | Optional View name that is used when a view on a non virtual is generated (default = [TASK-ID]_generated_view). | | The identifier for this plugin is `GenerateSparkView`. It can be found in the package `com.eccenca.di.sql.spark.virtual`. - - - diff --git a/docs/deploy-and-configure/configuration/dataintegration/index.md b/docs/deploy-and-configure/configuration/dataintegration/index.md index 7eb075fb..3c8fc0c2 100644 --- a/docs/deploy-and-configure/configuration/dataintegration/index.md +++ b/docs/deploy-and-configure/configuration/dataintegration/index.md @@ -354,6 +354,7 @@ workspace.repository.projectS3 = { # /path/to/my-workspace/ } ``` + For this S3 plugin make sure the account has at least these permissions attached: ``` json @@ -914,7 +915,7 @@ Within Build (DataIntegration) the SqlEndpoint can be used as a source or sink l Any JDBC or ODBC client can connect to a JDBC endpoint provided by an SqlEndpoint dataset. SqlEndpoint uses the same query processing as Hive, therefore the requirements for the client are: - A JDBC driver compatible with *Hive 1.2.1* (platform independent driver org.apache.hive.jdbc.HiveDriver is needed) or - - Hive 1.2.1 is [ODPi](https://github.com/odpi/specs/blob/master/ODPi-Runtime.md) runtime compliant + - Hive 1.2.1 is [ODPi](https://github.com/odpi/specs/blob/master/ODPi-Runtime.md) runtime compliant - A JDBC driver compatible with *Spark 2.3.3* - A Hive ODBC driver (ODBC driver for the client architecture and operating system needed) @@ -1036,10 +1037,10 @@ com.eccenca.di.mappingCreatorEnabled = true The [Mapping Creator can optionally use LLM](../../../build/mapping-creator/index.md#smart-suggestions-with-ai-support) to automatically generate class and property mappings. Use the following configuration section as a blueprint to set up your OpenAI-compatible endpoint, providing: -- the API key, -- a model, -- reasoning level, -- and (optionally) benchmarking outputs. +- the API key, +- a model, +- reasoning level, +- and (optionally) benchmarking outputs. ```bash linenums="1" ################################################# diff --git a/docs/deploy-and-configure/configuration/dataintegration/jdbc/index.md b/docs/deploy-and-configure/configuration/dataintegration/jdbc/index.md index 2ce2012c..2106b99c 100644 --- a/docs/deploy-and-configure/configuration/dataintegration/jdbc/index.md +++ b/docs/deploy-and-configure/configuration/dataintegration/jdbc/index.md @@ -20,10 +20,10 @@ You can also add and use custom drivers. The platform includes the following JDBC drivers: -- PostgreSQL (`postgresql v42.7.10`) -- MariaDB (includes support for MySQL, `mariadb-java-client v3.5.7`) -- Microsoft SQL Server (`mssql-jdbc v13.2.1.jre11`) -- Snowflake (`snowflake-jdbc v3.28.0`) +- PostgreSQL (`postgresql v42.7.10`) +- MariaDB (includes support for MySQL, `mariadb-java-client v3.5.7`) +- Microsoft SQL Server (`mssql-jdbc v13.2.1.jre11`) +- Snowflake (`snowflake-jdbc v3.28.0`) ## Custom JDBC Drivers @@ -40,10 +40,10 @@ Download the JDBC driver for each database management system that you want to co Consult your solutions manager or DevOps specialist for options to copy or inject the JDBC driver `jar` into a Corporate Memory deployment. Depending on the deployment model, suitable options include: -- The Docker Compose package `cmem-orchestration` mounts the folder `./conf/dataintegration/plugin/` into the DataIntegration container. +- The Docker Compose package `cmem-orchestration` mounts the folder `./conf/dataintegration/plugin/` into the DataIntegration container. The configuration snippets below assume this location, which maps to `/opt/cmem/eccenca-DataIntegration/dist/etc/dataintegration/conf/plugin/` inside the container. -- A dedicated _Build project_ in which the driver JAR files are uploaded as project file resources. -- Dedicated file or resource mounts in a Docker Compose or Helm/Kubernetes configuration. +- A dedicated _Build project_ in which the driver JAR files are uploaded as project file resources. +- Dedicated file or resource mounts in a Docker Compose or Helm/Kubernetes configuration. ### Driver Registration diff --git a/docs/deploy-and-configure/configuration/docker-orchestration/index.md b/docs/deploy-and-configure/configuration/docker-orchestration/index.md index 98c8b0e1..04f4efe5 100644 --- a/docs/deploy-and-configure/configuration/docker-orchestration/index.md +++ b/docs/deploy-and-configure/configuration/docker-orchestration/index.md @@ -18,7 +18,7 @@ The environment files are supplied in the CONFIGFILE environment variable to th For example, in [Scenario: Single Node Cloud Installation](../../installation/scenario-single-node-cloud-installation/index.md) we have created a `prod.env` environment file and created the Corporate Memory instance using `prod.env` configuration: ``` shell-session -$ CONFIGFILE=environments/prod.env make clean-pull-start-bootstrap +CONFIGFILE=environments/prod.env make clean-pull-start-bootstrap ``` When you run `make clean-pull-start-bootstrap` target, the Makefile will evaluate and export the environment variables from the `environments/default.env`, your `${CONFIGFILE}` or `environments/config.env` and `environments/scripted-env.mk`: @@ -43,18 +43,18 @@ To configure the orchestration according to your requirements, you need simply t For example, to replicate the minimum configuration from `config.env`, you can do the following: ``` shell-session -$ echo "create empty environments/prod.env file" -$ touch environments/prod.env -$ echo "inject necessary variables into the prod.env" -$ echo "CMEM_SERVICE_ACCOUNT_CLIENT_SECRET=c8c12828-000c-467b-9b6d-2d6b5e16df4a" >> environments/prod.env -$ echo "STARDOG_PASSWORD=admin" >> environments/prod.env -$ echo "TRUSTSTOREPASS=Aimeik5Ocho5riuC" >> environments/prod.env +echo "create empty environments/prod.env file" +touch environments/prod.env +echo "inject necessary variables into the prod.env" +echo "CMEM_SERVICE_ACCOUNT_CLIENT_SECRET=c8c12828-000c-467b-9b6d-2d6b5e16df4a" >> environments/prod.env +echo "STARDOG_PASSWORD=admin" >> environments/prod.env +echo "TRUSTSTOREPASS=Aimeik5Ocho5riuC" >> environments/prod.env ``` This configuration will be sufficient to run the orchestration locally as described in [Scenario: Local Installation](../../installation/scenario-local-installation/index.md): ``` shell-session -$ CONFIGFILE=environments/prod.env make clean-pull-start-bootstrap +CONFIGFILE=environments/prod.env make clean-pull-start-bootstrap ``` ## Available Configuration Variables @@ -81,7 +81,7 @@ All available configuration environment variables are listed in `environments/de | APACHE_CONFIG | default.conf | Apache2 virtual host configuration | | SSLCONF | ssl.default.conf | Apache2 virtual host configuration for SSL setup | | HTTP_PORT | 80 | APACHE_HTTP_PORT is used as a standard port 80 in SSL setup | -| LETSENCRYPT_MAIL | administration@eccenca.com | email to be used when requesting letsencrypt certificates | +| LETSENCRYPT_MAIL | | email to be used when requesting letsencrypt certificates | | DATAINTEGRATION_BASE_FILE | docker-compose.dataintegration-base.yml | `docker compose` extension file for Build (DataIntegration), see SSL configuration section below for an example | | TRUSTSTOREPASS | (empty) | Truststore password, see self-signed certificates configuration section below for an example | diff --git a/docs/deploy-and-configure/configuration/explore/dataplatform/application-full.md b/docs/deploy-and-configure/configuration/explore/dataplatform/application-full.md index feb890e5..0d6966b7 100644 --- a/docs/deploy-and-configure/configuration/explore/dataplatform/application-full.md +++ b/docs/deploy-and-configure/configuration/explore/dataplatform/application-full.md @@ -5,7 +5,6 @@ tags: ## Deployment options for explore container - ***Property: deploy.apiPrefix*** API prefix for former dataplatform endpoints i.e. /dataplatform @@ -30,10 +29,8 @@ URI where to redirect to when the user logs out ### Options for additional prometheus metrics endpoint - ***Property: deploy.additional-prometheus-endpoint.enabled*** - | Category | Value | |--- | ---: | | Default | false | @@ -43,7 +40,6 @@ URI where to redirect to when the user logs out ***Property: deploy.additional-prometheus-endpoint.port*** - | Category | Value | |--- | ---: | | Default | 9091 | @@ -53,7 +49,6 @@ URI where to redirect to when the user logs out ***Property: deploy.additional-prometheus-endpoint.context*** - | Category | Value | |--- | ---: | | Default | /metrics | @@ -73,12 +68,10 @@ In case a dedicated license file is used, different configuration options can ov 3. license.asc file in the same folder, where the application is started from (in Standalone Mode) 4. Fallback to eccenca free Personal, Evaluation and Development License Agreement (PEDAL) - ***Property: license.key*** Use this property to specify the license key as a YAML multiline string value of the license.key property. - ```yaml key: | -----BEGIN PGP MESSAGE----- @@ -87,13 +80,12 @@ key: | -----END PGP MESSAGE----- ``` - | Category | Value | |--- | ---: | | Default | *none* | | Required | false | | Valid values | PGP Key (Message) | - | Conflicts with | license.file | +| Conflicts with | license.file | | Environment | LICENSE_KEY | ***Property: license.file*** @@ -105,14 +97,13 @@ Use this property to specify the location of the license file | Default | *none* | | Required | false | | Valid values | location of the license file | - | Conflicts with | license.key | +| Conflicts with | license.key | | Environment | LICENSE_FILE | ## General platform settings for DataPlatform This section provides general configuration settings. - ### Configuration of Caching DataPlatform provides caching support which is enabled by default with an in-memory Caffeine cache. @@ -124,7 +115,6 @@ which can be further configured under the custom properties "spring.cache.infini To disable caching, set the type to NONE (not recommended). - | Category | Value | |--- | ---: | | Default | INFINISPAN | @@ -134,7 +124,6 @@ To disable caching, set the type to NONE (not recommended). ***Property: spring.cache.infinispan.mode*** - | Category | Value | |--- | ---: | | Default | LOCAL | @@ -144,7 +133,6 @@ To disable caching, set the type to NONE (not recommended). ***Property: spring.mvc.pathmatch.matching-strategy*** - | Category | Value | |--- | ---: | | Default | ant_path_matcher | @@ -154,7 +142,6 @@ To disable caching, set the type to NONE (not recommended). ***Property: spring.thymeleaf.prefix*** - | Category | Value | |--- | ---: | | Default | classpath:/public/ | @@ -164,7 +151,6 @@ To disable caching, set the type to NONE (not recommended). ***Property: spring.thymeleaf.mode*** - | Category | Value | |--- | ---: | | Default | HTML | @@ -178,7 +164,6 @@ DataSource configuration for the chat memory persistence using HSQLDB. In production/Docker environments, data is stored in /var/lib/explore/chat-memory. In development environments, this can be overridden in application-dev.yml. - ***Property: spring.datasource.driver-class-name*** JDBC driver class for HSQLDB @@ -225,19 +210,16 @@ Database password for HSQLDB (empty for embedded mode) ### Configuration of Servlet Container - Multipart upload limits config You may need to set the following parameter values to 2048MB for implementations that cannot handle large requests - ***Property: spring.servlet.multipart.max-file-size*** Use this property to define the maximum size of an uploaded file in number of bytes. Values can use the suffixed "MB" or "KB" (e.g. '1024MB'). **Note:** If DataPlatform is deployed in a Servlet container, make sure to also configure support for large file sizes. - | Category | Value | |--- | ---: | | Default | 4096MB | @@ -269,7 +251,6 @@ Temporary storage used for multipart upload. This defaults to system property ja ***Property: spring.jackson.default-property-inclusion*** - | Category | Value | |--- | ---: | | Default | non_null | @@ -279,7 +260,6 @@ Temporary storage used for multipart upload. This defaults to system property ja ***Property: spring.ai.retry.max-attempts*** - | Category | Value | |--- | ---: | | Default | 2 | @@ -291,7 +271,6 @@ Temporary storage used for multipart upload. This defaults to system property ja Use this property to enable or disable the AI capabilities of Explore - | Category | Value | |--- | ---: | | Default | false | @@ -331,7 +310,7 @@ Server version ***Property: spring.ai.mcp.server.type*** -Server type (SYNC/ASYNC) s. https://docs.spring.io/spring-ai/reference/index.html +Server type (SYNC/ASYNC) s. | Category | Value | |--- | ---: | @@ -342,7 +321,6 @@ Server type (SYNC/ASYNC) s. https://docs.spring.io/spring-ai/reference/index.htm ***Property: spring.ai.mcp.server.instructions*** - | Category | Value | |--- | ---: | | Default | This server provides access to corporate memory | @@ -352,7 +330,6 @@ Server type (SYNC/ASYNC) s. https://docs.spring.io/spring-ai/reference/index.htm ***Property: spring.ai.mcp.server.capabilities.tool*** - | Category | Value | |--- | ---: | | Default | true | @@ -362,7 +339,6 @@ Server type (SYNC/ASYNC) s. https://docs.spring.io/spring-ai/reference/index.htm ***Property: spring.ai.mcp.server.capabilities.resource*** - | Category | Value | |--- | ---: | | Default | true | @@ -372,7 +348,6 @@ Server type (SYNC/ASYNC) s. https://docs.spring.io/spring-ai/reference/index.htm ***Property: spring.ai.mcp.server.capabilities.prompt*** - | Category | Value | |--- | ---: | | Default | true | @@ -382,7 +357,6 @@ Server type (SYNC/ASYNC) s. https://docs.spring.io/spring-ai/reference/index.htm ***Property: spring.ai.mcp.server.capabilities.completion*** - | Category | Value | |--- | ---: | | Default | true | @@ -403,7 +377,7 @@ Whether the mcp server is enabled or not ***Property: spring.ai.mcp.server.protocol*** -Protocol used for communication between mcp server and client s. https://docs.spring.io/spring-ai/reference/index.html +Protocol used for communication between mcp server and client s. | Category | Value | |--- | ---: | @@ -449,7 +423,6 @@ Endpoint for mcp sse protocol (prefix must be dataplatform) This configures the possible chat models for interacting with the companion. - | Category | Value | |--- | ---: | | Default | openai | @@ -459,7 +432,6 @@ This configures the possible chat models for interacting with the companion. ***Property: spring.ai.model.embedding*** - | Category | Value | |--- | ---: | | Default | openai | @@ -469,7 +441,6 @@ This configures the possible chat models for interacting with the companion. ***Property: spring.ai.anthropic.api-key*** - | Category | Value | |--- | ---: | | Default | | @@ -479,7 +450,6 @@ This configures the possible chat models for interacting with the companion. ***Property: spring.ai.anthropic.chat.options.max-tokens*** - | Category | Value | |--- | ---: | | Default | 10000 | @@ -489,7 +459,6 @@ This configures the possible chat models for interacting with the companion. ***Property: spring.ai.openai.api-key*** - | Category | Value | |--- | ---: | | Default | | @@ -499,7 +468,6 @@ This configures the possible chat models for interacting with the companion. ***Property: spring.ai.openai.chat.options.response-format.type*** - | Category | Value | |--- | ---: | | Default | json_schema | @@ -509,10 +477,10 @@ This configures the possible chat models for interacting with the companion. ***Property: spring.ai.openai.chat.options.response-format.schema*** - | Category | Value | |--- | ---: | -| Default | { \"name\": \"string\", \"schema\": \"string\" } +| Default | { \"name\": \"string\", \"schema\": \"string\" } | + | | Required | false | | Valid values | string | @@ -520,7 +488,6 @@ This configures the possible chat models for interacting with the companion. ***Property: spring.ai.azure.openai.endpoint*** - | Category | Value | |--- | ---: | | Default | | @@ -530,7 +497,6 @@ This configures the possible chat models for interacting with the companion. ***Property: spring.ai.azure.openai.chat.options.response-format.type*** - | Category | Value | |--- | ---: | | Default | json_schema | @@ -540,10 +506,10 @@ This configures the possible chat models for interacting with the companion. ***Property: spring.ai.azure.openai.chat.options.response-format.schema*** - | Category | Value | |--- | ---: | -| Default | { \"name\": \"string\", \"schema\": \"string\" } +| Default | { \"name\": \"string\", \"schema\": \"string\" } | + | | Required | false | | Valid values | string | @@ -551,7 +517,6 @@ This configures the possible chat models for interacting with the companion. ***Property: spring.ai.mistral-ai.api-key*** - | Category | Value | |--- | ---: | | Default | | @@ -561,7 +526,6 @@ This configures the possible chat models for interacting with the companion. ***Property: management.info.env.enabled*** - | Category | Value | |--- | ---: | | Default | true | @@ -571,7 +535,6 @@ This configures the possible chat models for interacting with the companion. ***Property: management.endpoints.web.base-path*** - | Category | Value | |--- | ---: | | Default | /dataplatform/actuator | @@ -581,7 +544,6 @@ This configures the possible chat models for interacting with the companion. ***Property: management.endpoints.web.exposure.include*** - | Category | Value | |--- | ---: | | Default | * | @@ -591,7 +553,6 @@ This configures the possible chat models for interacting with the companion. ***Property: management.endpoints.enabled-by-default*** - | Category | Value | |--- | ---: | | Default | false | @@ -601,7 +562,6 @@ This configures the possible chat models for interacting with the companion. ***Property: management.endpoint.health.enabled*** - | Category | Value | |--- | ---: | | Default | true | @@ -611,7 +571,6 @@ This configures the possible chat models for interacting with the companion. ***Property: management.endpoint.health.show-details*** - | Category | Value | |--- | ---: | | Default | when_authorized | @@ -621,7 +580,6 @@ This configures the possible chat models for interacting with the companion. ***Property: management.endpoint.info.enabled*** - | Category | Value | |--- | ---: | | Default | true | @@ -631,7 +589,6 @@ This configures the possible chat models for interacting with the companion. ***Property: management.health.diskspace.enabled*** - | Category | Value | |--- | ---: | | Default | false | @@ -641,7 +598,6 @@ This configures the possible chat models for interacting with the companion. ***Property: management.health.livenessstate.enabled*** - | Category | Value | |--- | ---: | | Default | true | @@ -651,7 +607,6 @@ This configures the possible chat models for interacting with the companion. ***Property: management.health.readinessstate.enabled*** - | Category | Value | |--- | ---: | | Default | true | @@ -661,7 +616,6 @@ This configures the possible chat models for interacting with the companion. ***Property: management.health.sparql.enabled*** - | Category | Value | |--- | ---: | | Default | true | @@ -671,7 +625,6 @@ This configures the possible chat models for interacting with the companion. ***Property: management.health.sparql.fixedDelayInMilliseconds*** - | Category | Value | |--- | ---: | | Default | 5000 | @@ -681,7 +634,6 @@ This configures the possible chat models for interacting with the companion. ***Property: management.health.sparql.timeoutInMilliseconds*** - | Category | Value | |--- | ---: | | Default | 5000 | @@ -691,7 +643,6 @@ This configures the possible chat models for interacting with the companion. ***Property: management.influx.metrics.export.enabled*** - | Category | Value | |--- | ---: | | Default | false | @@ -732,7 +683,6 @@ springdoc: enabled: true ``` - ***Property: springdoc.api-docs.enabled*** Use this property to enable and expose endpoint that provide the OpenAPI compliant specification of the DataPlatform APIs. The following endpoints will become available when this option is set to true: @@ -741,7 +691,6 @@ Use this property to enable and expose endpoint that provide the OpenAPI complia - /v3/api-docs.yaml - /v3/api-docs/swagger-config - | Category | Value | |--- | ---: | | Default | false | @@ -755,7 +704,6 @@ Use this property to enable and expose a Swagger UI browser interface that can b - /swagger-ui.html - | Category | Value | |--- | ---: | | Default | false | @@ -777,10 +725,9 @@ http: - https://example.com ``` - ***Property: http.cors.allowedOrigins*** -Use this property to define the list of allowed origins. The values must be either specific origins, e.g. http://example.org, or * for all origins. +Use this property to define the list of allowed origins. The values must be either specific origins, e.g. , or * for all origins. | Category | Value | |--- | ---: | @@ -848,7 +795,6 @@ Use this property to define how long in seconds the response from a pre-flight r Java 11 HTTP client settings for HTTP access to the backend store. - ***Property: httpclient.connectionPoolSize*** The maximum number of connections to keep in the HTTP/1.1 keep alive cache. A value of 0 means that the cache is unbounded @@ -889,7 +835,6 @@ authorization: adminGroup: admins ``` - | Category | Value | |--- | ---: | | Default | true | @@ -939,10 +884,9 @@ Use this property to configure the group that gets root access if enabled (see s Use this property to configure the URI of the public user group (see section Public access). **Note:** If you change this property, you also need to change existing URI descriptions and existing access conditions. - | Category | Value | |--- | ---: | -| Default | https://vocab.eccenca.com/auth/PublicGroup | +| Default | | | Required | false | | Valid values | string | | Environment | AUTHORIZATION_ABOX_PUBLICGROUP | @@ -952,22 +896,20 @@ Use this property to configure the URI of the public user group (see section Pub Use this property to configure the URI of the public user (see section Public access). **Note:** If you change this property, you also need to change existing URI descriptions and existing access conditions. - | Category | Value | |--- | ---: | -| Default | https://vocab.eccenca.com/auth/AnonymousUser | +| Default | | | Required | false | | Valid values | string | | Environment | AUTHORIZATION_ABOX_ANONYMOUSUSER | -#### Access conditions +### Access conditions **IMPORTANT:** The following properties are deprecated and have no function anymore! - ***Property: authorization.abox.accessConditions.url*** -**Deprecation:** Use this property to set the URL of the access conditions model file. This can be either a remote (http://...) or a local (file:...) .rdf file. Refer to section Access conditions for more information on the access conditions model. +**Deprecation:** Use this property to set the URL of the access conditions model file. This can be either a remote (...) or a local (file:...) .rdf file. Refer to section Access conditions for more information on the access conditions model. | Category | Value | @@ -982,13 +924,12 @@ Use this property to configure the URI of the public user (see section Public ac **Deprecation:** Use this property to set the graph containing the access conditions model. **Note:** If you change this property, you also need to change the corresponding shape definitions for access conditions (more precisely, the UI SPARQL queries). - | Category | Value | |--- | ---: | -| Default | https://ns.eccenca.com/data/ac/ | +| Default | | | Required | false | | Valid values | string | - | Conflicts with | url | +| Conflicts with | url | | Environment | AUTHORIZATION_ABOX_ACCESSCONDITIONS_GRAPH | ## SPARQL endpoints @@ -1013,14 +954,13 @@ proxy: - "" ``` - ***Property: proxy.defaultBaseIri*** -Base IRI for this Corporate Memory instance. If not set falls back to environment variable DEPLOY_BASE_URL, further fallback to https://fallback.eccenca.com/ +Base IRI for this Corporate Memory instance. If not set falls back to environment variable DEPLOY_BASE_URL, further fallback to | Category | Value | |--- | ---: | -| Default | https://fallback.eccenca.com/ | +| Default | | | Required | false | | Valid values | URI | | Environment | PROXY_DEFAULTBASEIRI | @@ -1030,10 +970,9 @@ Base IRI for this Corporate Memory instance. If not set falls back to environmen Use this property to specify which RDF properties should be used to provide label values when matching IRIs against a search term during rewriting SELECT-queries. **Note:** This configuration property affects modification of SELECT-queries for search triggered by the search-string query parameter. Results of SELECT-queries when the resolveLabels property is set to LABELS - | Category | Value | |--- | ---: | -| Default | [http://www.w3.org/2004/02/skos/core#prefLabel, http://www.w3.org/2000/01/rdf-schema#label, http://purl.org/dc/terms/title, http://www.w3.org/ns/shacl#name] | +| Default | [, , , ] | | Required | false | | Valid values | list of Properties | | Environment | PROXY_LABELPROPERTIES | @@ -1043,10 +982,9 @@ Use this property to specify which RDF properties should be used to provide labe Use this property to specify which RDF properties should be used to provide description values when matching IRIs against a search term during rewriting SELECT-queries. **Note:** This configuration property affects modification of SELECT-queries for search triggered by the search-string query parameter. Results of SELECT-queries when the resolveLabels property is set to LABELS - | Category | Value | |--- | ---: | -| Default | [http://www.w3.org/2004/02/skos/core#definition, http://purl.org/dc/terms/description, http://www.w3.org/2000/01/rdf-schema#comment] | +| Default | [, , ] | | Required | false | | Valid values | list of Properties | | Environment | PROXY_DESCRIPTIONPROPERTIES | @@ -1057,7 +995,6 @@ Specifies base language preferences for this instance. **Note:** This configuration property affects results of SELECT-queries when the resolveLabels property is set to LABELS. - | Category | Value | |--- | ---: | | Default | [en, , de, fr] | @@ -1082,7 +1019,6 @@ The Concise Boundary Description is used for viewing and editing resoures. By default up to a max of 5 Blank nodes are traversed for calculation. Increasing the max fetch will support deeper constructs, but will also add to loading time. - | Category | Value | |--- | ---: | | Default | 5 | @@ -1095,7 +1031,6 @@ Increasing the max fetch will support deeper constructs, but will also add to lo The max amount of statements which the Concise Bound Description can contain. (S)CBDs surpassing this will not load but return an error - | Category | Value | |--- | ---: | | Default | 1000000 | @@ -1124,7 +1059,6 @@ returned per `shacl:PropertyShape`. The default needs to be larger than the Data for 'propertyLimit', which is up to 25. Changing this value allows custom endpoints to fetch more data. Increasing this value will increase response time - | Category | Value | |--- | ---: | | Default | 26 | @@ -1192,10 +1126,10 @@ Maximum amount of memory entries for shacl batch validation results can take up. Value Fetch Strategy Determines how the Knowledge Graph is walked for values for specific resources. Used for resolving titles & comments and loading shaped resources. + - RESOURCE_IN_VALUES uses a SPARQL `VALUES (?resource ) { (:resource1)(:resource2)}` - FILTER_ONLY Uses SPARQL uses a SPARQL `FILTER (?resource in (:resource1, :resource2))` - | Category | Value | |--- | ---: | | Default | RESOURCE_IN_VALUES | @@ -1263,7 +1197,6 @@ gitSync: scheduledPullCron: "0 */5 * * * *" ``` - ***Property: gitSync.enabled*** Activates / Deactivates git graph sync feature @@ -1347,7 +1280,7 @@ The committer email which appears in the commit message on system commits | Category | Value | |--- | ---: | -| Default | info@eccenca.com | +| Default | | | Required | false | | Valid values | string | | Environment | GITSYNC_COMMITTEREMAIL | @@ -1360,7 +1293,7 @@ Schedules Pull Frequency - Configured git repositories for sync are pulled regul |--- | ---: | | Default | 0 */30 * * * * | | Required | false | -| Valid values | Cron setting according to https://docs.spring.io/spring-framework/docs/current/reference/html/integration.html#scheduling-cron-expression | +| Valid values | Cron setting according to | | Environment | GITSYNC_SCHEDULEDPULLCRON | ***Property: semspect.enabled*** @@ -1399,11 +1332,9 @@ logging: file: /var/logs/dataplatform.log ``` - Use these properties to specify where you want to store your logging file. Specifying a file leads to both, logging to standard output and the file. File output creates an auto-rotating file with 10 MB file size each. - ***Property: logging.file.name*** Log file name (for instance, `myapp.log`). Names can be an exact location or relative to the current directory. @@ -1437,7 +1368,6 @@ logging: configuration: ELDS_HOME/etc/dataplatform/logback.xml ``` - | Category | Value | |--- | ---: | | Default | *none* | @@ -1447,7 +1377,6 @@ logging: ***Property: logging.level.audit*** - | Category | Value | |--- | ---: | | Default | INFO | @@ -1457,7 +1386,6 @@ logging: ***Property: logging.level.com.eccenca.elds.backend*** - | Category | Value | |--- | ---: | | Default | INFO | @@ -1467,7 +1395,6 @@ logging: ***Property: logging.level.org.springframework*** - | Category | Value | |--- | ---: | | Default | WARN | @@ -1477,7 +1404,6 @@ logging: ***Property: logging.level.com.eccenca.elds.backend.webapp.web.filter.SimpleCorsFilter*** - | Category | Value | |--- | ---: | | Default | WARN | @@ -1487,7 +1413,6 @@ logging: ***Property: logging.level.com.eccenca.elds.backend.webapp.web.GlobalControllerExceptionHandler*** - | Category | Value | |--- | ---: | | Default | TRACE | @@ -1497,7 +1422,6 @@ logging: ***Property: logging.level.com.eccenca.elds.backend.cache.logging*** - | Category | Value | |--- | ---: | | Default | WARN | @@ -1507,7 +1431,6 @@ logging: ***Property: logging.level.org.hibernate.search.backend.lucene.impl*** - | Category | Value | |--- | ---: | | Default | ERROR | @@ -1527,13 +1450,11 @@ auditTrail: - "aksw.org" ``` - ***Property: audit-trail.enabled*** Use this property to enable logging of read and write access to every graph access. If auditTrail.auditedGraphs is specified, only those graphs are logged. **Note:** If audit trail logging is enabled, RDF upload over the Graph Store Protocol interface is limited to triple formats. Any attempt to upload a quad format results in an HTTP 415 error. - | Category | Value | |--- | ---: | | Default | false | @@ -1556,7 +1477,6 @@ Limits the size of the query response ***Property: sparql.query.limit*** - | Category | Value | |--- | ---: | | Default | 100000 | @@ -1581,7 +1501,6 @@ server: contextPath: /dataplatform ``` - ***Property: server.port*** Use this property to set the TCP port where the embedded server is available. @@ -1595,7 +1514,6 @@ Use this property to set the TCP port where the embedded server is available. ***Property: server.error.include-stacktrace*** - | Category | Value | |--- | ---: | | Default | NEVER | @@ -1618,7 +1536,6 @@ Tomcat servlet settings ***Property: server.servlet.session.cookie.same-site*** - | Category | Value | |--- | ---: | | Default | Lax | @@ -1642,7 +1559,6 @@ server: client-auth: NEED ``` - ***Property: server.ssl.key-store*** Use this property to define the path to the KeyStore used for one-way or two-way SSL authentication. @@ -1653,7 +1569,6 @@ In case of two-way authentication, a TrustStore must also be configured. This co JAVA_TOOL_OPTIONS=-Djavax.net.ssl.trustStore=path_to_trust_store.jks -Djavax.net.ssl.trustStorePassword=trust_store_password (ADD TO EXISTING JAVA_TOOL_OPTIONS) ``` - | Category | Value | |--- | ---: | | Default | *none* | @@ -1678,7 +1593,6 @@ Use this property to define the client identification policy. If WANT is set, client identification is optional. If NEED is set, client identification is mandatory, so unauthenticated clients are refused. - | Category | Value | |--- | ---: | | Default | *none* | @@ -1704,7 +1618,6 @@ Configuration recommendation: **Note:** This configuration recommendation provides settings for headers most commonly used by proxies. Make sure to add all three properties in order to enforce HTTPS. - ***Property: server.tomcat.remoteIpHeader*** Use this property to set the request header which is required to identify the originating IP address of the client connecting to DataPlatform through an HTTP proxy. @@ -1729,7 +1642,6 @@ Use this property to set the request header which is required to identify the or ***Property: server.tomcat.max-swallow-size*** - | Category | Value | |--- | ---: | | Default | -1 | @@ -1779,13 +1691,11 @@ Limits how many background query requests can be run in parallel. This applies t Files can be asynchronously uploaded to the backend store in multiple steps which include an analysis of the uploaded file. Please s. API documentation under /api/upload/ for further information. - ***Property: files.maxStorageSingleFileSizeMb*** Maximum size of one stored file (as uploaded i.e. can also be compressed size) Value in Mb - | Category | Value | |--- | ---: | | Default | 3000 | @@ -1798,7 +1708,6 @@ Value in Mb Minimum storage space left on temp device of DataPlatform for file uploads Value in Mb - | Category | Value | |--- | ---: | | Default | 3000 | @@ -1811,7 +1720,6 @@ Value in Mb Cron setting for housekeeping / maintenance job Stored files and saved analysis will be deleted if older than maintenanceExpirationDuration - | Category | Value | |--- | ---: | | Default | P1D | @@ -1877,4 +1785,3 @@ Query timeout as duration which is active if no timeout in request has been set | Required | false | | Valid values | ISO 8601 duration format | | Environment | STORE_QUERYTIMEOUTGENERAL | - diff --git a/docs/deploy-and-configure/configuration/explore/dataplatform/application-graphdb-full.md b/docs/deploy-and-configure/configuration/explore/dataplatform/application-graphdb-full.md index 257eeaab..c89c31d3 100644 --- a/docs/deploy-and-configure/configuration/explore/dataplatform/application-graphdb-full.md +++ b/docs/deploy-and-configure/configuration/explore/dataplatform/application-graphdb-full.md @@ -27,7 +27,6 @@ store: createRepositoryOnStartup: true ``` - ***Property: store.type*** The type of the store must be set to "graphdb" @@ -41,7 +40,6 @@ The type of the store must be set to "graphdb" ### Specific settings for GraphDB - ***Property: store.graphdb.host*** The host of the GraphDB database @@ -158,7 +156,7 @@ The iri of the lucene index to be used for searches. If the default index is use | Category | Value | |--- | ---: | -| Default | http://www.ontotext.com/connectors/lucene/instance#cmembaseindex | +| Default | | | Required | false | | Valid values | Valid URI of lucene index | | Environment | STORE_GRAPHDB_GDBBASEINDEX | @@ -184,4 +182,3 @@ Maximum amount of quads of change tracking result which will be loaded in memory | Required | false | | Valid values | int | | Environment | STORE_GRAPHDB_GRAPHDBCHANGETRACKINGMAXQUADMEMORY | - diff --git a/docs/deploy-and-configure/configuration/explore/dataplatform/application-http-full.md b/docs/deploy-and-configure/configuration/explore/dataplatform/application-http-full.md index 921f665e..419f03ca 100644 --- a/docs/deploy-and-configure/configuration/explore/dataplatform/application-http-full.md +++ b/docs/deploy-and-configure/configuration/explore/dataplatform/application-http-full.md @@ -22,7 +22,6 @@ store: password: "password" ``` - ***Property: store.type*** The type of the store must be set to "http" @@ -36,7 +35,6 @@ The type of the store must be set to "http" ***Property: store.authorization*** - | Category | Value | |--- | ---: | | Default | REWRITE_FROM | @@ -52,7 +50,7 @@ Use this property to configure the endpoint to which SPARQL 1.1 queries are sent | Category | Value | |--- | ---: | -| Default | http://localhost:7200/repositories/cmem | +| Default | | | Required | true | | Valid values | string | | Environment | STORE_HTTP_QUERY_ENDPOINT_URL | @@ -63,7 +61,7 @@ Use this property to configure the endpoint to which SPARQL 1.1 updates are sent | Category | Value | |--- | ---: | -| Default | http://localhost:7200/repositories/cmem/statements | +| Default | | | Required | true | | Valid values | string | | Environment | STORE_HTTP_UPDATE_ENDPOINT_URL | @@ -74,7 +72,7 @@ Use this property to configure the endpoint to SPARQL 1.1 Graph Store Protocol r | Category | Value | |--- | ---: | -| Default | http://localhost:7200/repositories/cmem/rdf-graphs/service | +| Default | | | Required | false | | Valid values | string | | Environment | STORE_HTTP_GRAPH_STORE_ENDPOINT_URL | @@ -122,4 +120,3 @@ Defines how the raw list of graphs is retrieved, and therefore which graphs are | Required | false | | Valid values | Valid SPARQL query with bound variable "g" | | Environment | STORE_HTTP_GRAPHLISTQUERY | - diff --git a/docs/deploy-and-configure/configuration/explore/dataplatform/application-inmemory-full.md b/docs/deploy-and-configure/configuration/explore/dataplatform/application-inmemory-full.md index 5918ea3f..28a965e0 100644 --- a/docs/deploy-and-configure/configuration/explore/dataplatform/application-inmemory-full.md +++ b/docs/deploy-and-configure/configuration/explore/dataplatform/application-inmemory-full.md @@ -21,7 +21,6 @@ store: - "/data/data.trig" ``` - ***Property: store.type*** The type of the store must be set to "memory" @@ -35,7 +34,6 @@ The type of the store must be set to "memory" ***Property: store.authorization*** - | Category | Value | |--- | ---: | | Default | REWRITE_FROM | @@ -55,4 +53,3 @@ list of files in file URI scheme | Required | false | | Valid values | A list of files | | Environment | STORE_MEMORY_FILES | - diff --git a/docs/deploy-and-configure/configuration/explore/dataplatform/application-neptune-full.md b/docs/deploy-and-configure/configuration/explore/dataplatform/application-neptune-full.md index 49ddcda0..d3c21fdb 100644 --- a/docs/deploy-and-configure/configuration/explore/dataplatform/application-neptune-full.md +++ b/docs/deploy-and-configure/configuration/explore/dataplatform/application-neptune-full.md @@ -34,7 +34,6 @@ store: management.tracing.enabled: false ``` - ***Property: store.type*** The type of the store must be set to "neptune" @@ -74,7 +73,7 @@ Settings for the connection to the Amazon Cloud ***Property: store.neptune.aws.region*** -The region where the neptune instance is located i.e. "eu-central-1" s. https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions +The region where the neptune instance is located i.e. "eu-central-1" s. | Category | Value | |--- | ---: | @@ -85,7 +84,7 @@ The region where the neptune instance is located i.e. "eu-central-1" s. https:// ***Property: store.neptune.aws.authEnabled*** -Whether the neptune instance is configured with enabled IAM authentication. In case of enabled authentication the credentials need to be accessible to the JVM of the dataplatform. Deployment on EC2 and assigning a role to the VM is sufficient. Other ways to achieve this are described in https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html +Whether the neptune instance is configured with enabled IAM authentication. In case of enabled authentication the credentials need to be accessible to the JVM of the dataplatform. Deployment on EC2 and assigning a role to the VM is sufficient. Other ways to achieve this are described in | Category | Value | |--- | ---: | @@ -94,7 +93,7 @@ Whether the neptune instance is configured with enabled IAM authentication. In c | Valid values | boolean | | Environment | STORE_NEPTUNE_AWS_AUTHENABLED | -Settings for S3 bucket connection and upload of large files to the neptune instance. The neptune store blocks all HTTP requests with size >150MB. To upload larger files a graph file is temporarily stored in a S3 bucket and uploaded via Neptune Bulk Loader. The S3 bucket needs to be in the same region as the neptune cluster. For more information s. https://docs.aws.amazon.com/neptune/latest/userguide/bulk-load.html. If no S3 upload is necessary then the limit of 150 MB on HTTPS uploads apply for neptune. The whole section can be left out of the configuration. +Settings for S3 bucket connection and upload of large files to the neptune instance. The neptune store blocks all HTTP requests with size >150MB. To upload larger files a graph file is temporarily stored in a S3 bucket and uploaded via Neptune Bulk Loader. The S3 bucket needs to be in the same region as the neptune cluster. For more information s. . If no S3 upload is necessary then the limit of 150 MB on HTTPS uploads apply for neptune. The whole section can be left out of the configuration. ***Property: store.neptune.s3.bucketNameOrAPAlias*** @@ -109,7 +108,7 @@ The name of the bucket or access point -> the role CMEM runs under needs write a ***Property: store.neptune.s3.iamRoleArn*** -The name of the role the neptune loader accesses the bucket -> the role needs read access to the bucket s. https://docs.aws.amazon.com/neptune/latest/userguide/bulk-load-tutorial-IAM.html +The name of the role the neptune loader accesses the bucket -> the role needs read access to the bucket s. | Category | Value | |--- | ---: | @@ -139,4 +138,3 @@ The degree of parallelism (CPU) for the neptune loader, possible values are LOW, | Required | false | | Valid values | LOW, MEDIUM, HIGH, OVERSUBSCRIBE | | Environment | STORE_NEPTUNE_S3_BULKLOADPARALLELISM | - diff --git a/docs/deploy-and-configure/configuration/explore/dataplatform/application-oauth-full.md b/docs/deploy-and-configure/configuration/explore/dataplatform/application-oauth-full.md index 844926c4..2e610ee2 100644 --- a/docs/deploy-and-configure/configuration/explore/dataplatform/application-oauth-full.md +++ b/docs/deploy-and-configure/configuration/explore/dataplatform/application-oauth-full.md @@ -20,14 +20,12 @@ spring: issuerUri: http://keycloak/auth/realms/cmem ``` - #### OAuth 2.0 Resource Server In order to protect access to it’s resources, DataPlatform acts as an OAuth 2.0 resource server accepting and responding to a protected resource request using a JSON Web Token (JWT). The OAuth 2.0 specification as well as the JSON Web Token specification don’t define any mandatory claims to be contained in a JWT access token. However, if the property spring.security.oauth2.resourceserver.jwt.issuer-uri is set, the iss (issuer) claim is required to be contained in the JWT. It’s value must be equal to the configured issuer URI. Additionally, in order to identify the requesting principal, either the username claim or the clientId claim must be contained in the JWT. - ***Property: spring.security.oauth2.resourceserver.anonymous*** Use this property to allow anonymous access to protected resources. @@ -46,10 +44,9 @@ If this property is set, the iss (issuer) claim is required to be contained in t **Note:** If the authorization server is down when DataPlatform queries it (given appropriate timeouts), then startup will fail. Also, if the authorization server doesn’t support the Provider Configuration endpoint, or if DataPlatform must be able to start up independently from the authorization server, use the property jwk-set-uri instead. - | Category | Value | |--- | ---: | -| Default | http://docker.localhost/auth/realms/cmem | +| Default | | | Required | false | | Valid values | URI to OpenID Connect Provider | | Conflicts with | spring.security.oauth2.resourceserver.jwt.jwkSetUri | @@ -88,7 +85,7 @@ Use this property to specify the claim identifying the roles (authorities) of th |--- | ---: | | Default | groups | | Required | false | -| Valid values | string | list of strings | +| Valid values | string | | Environment | SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_CLAIMS_GROUPS | ***Property: spring.security.oauth2.resourceserver.jwt.claims.clientId*** @@ -105,15 +102,13 @@ Use this property to specify the claim providing the OAuth 2.0 client ID to whic #### OAuth 2.0 client configuration In order to protect access to it's resources, DataPlatform acts as an OAuth 2.0 Client which provides authentication its own clients by means of a session cookie. For this type of authentication a JSON Web Token (JWT) -is not necessary. The registration which is configured is named "keycloak" and provides a login page redirecting to a keycloak backend. For specific customizations please s. https://docs.spring.io/spring-security/reference/servlet/oauth2/client/index.html +is not necessary. The registration which is configured is named "keycloak" and provides a login page redirecting to a keycloak backend. For specific customizations please s. One authentication backend is configured named 'keycloak'. The login page is accessible under '{basepath}/oauth2/authorization/keycloak' - ***Property: spring.security.oauth2.client.registration.keycloak.client-id*** - | Category | Value | |--- | ---: | | Default | dataintegration | @@ -123,7 +118,6 @@ One authentication backend is configured named 'keycloak'. The login page is acc ***Property: spring.security.oauth2.client.registration.keycloak.authorization-grant-type*** - | Category | Value | |--- | ---: | | Default | authorization_code | @@ -133,7 +127,6 @@ One authentication backend is configured named 'keycloak'. The login page is acc ***Property: spring.security.oauth2.client.registration.keycloak.client-authentication-method*** - | Category | Value | |--- | ---: | | Default | basic | @@ -143,7 +136,6 @@ One authentication backend is configured named 'keycloak'. The login page is acc ***Property: spring.security.oauth2.client.registration.keycloak.redirectUri*** - | Category | Value | |--- | ---: | | Default | {baseUrl}/login/oauth2/code/{registrationId} | @@ -153,7 +145,6 @@ One authentication backend is configured named 'keycloak'. The login page is acc ***Property: spring.security.oauth2.client.registration.keycloak.scope*** - | Category | Value | |--- | ---: | | Default | [openid, profile, email] | @@ -163,21 +154,18 @@ One authentication backend is configured named 'keycloak'. The login page is acc ***Property: spring.security.oauth2.client.registration.keycloak.provider.keycloak.issuer-uri*** - | Category | Value | |--- | ---: | -| Default | http://docker.localhost/auth/realms/cmem | +| Default | | | Required | false | | Valid values | string | | Environment | SPRING_SECURITY_OAUTH2_CLIENT_REGISTRATION_KEYCLOAK_PROVIDER_KEYCLOAK_ISSUER_URI | ***Property: spring.security.oauth2.client.registration.keycloak.provider.keycloak.user-name-attribute*** - | Category | Value | |--- | ---: | | Default | preferred_username | | Required | false | | Valid values | string | | Environment | SPRING_SECURITY_OAUTH2_CLIENT_REGISTRATION_KEYCLOAK_PROVIDER_KEYCLOAK_USER_NAME_ATTRIBUTE | - diff --git a/docs/deploy-and-configure/configuration/explore/dataplatform/application-virtuoso-full.md b/docs/deploy-and-configure/configuration/explore/dataplatform/application-virtuoso-full.md index b72f8a48..873286ce 100644 --- a/docs/deploy-and-configure/configuration/explore/dataplatform/application-virtuoso-full.md +++ b/docs/deploy-and-configure/configuration/explore/dataplatform/application-virtuoso-full.md @@ -7,7 +7,7 @@ tags: Configuration example: -This example configures a connection with HTTPS to a remote Virtuoso store (https://remote:8080). +This example configures a connection with HTTPS to a remote Virtuoso store (). ```yaml store: @@ -22,7 +22,6 @@ store: databasePort: 1111 ``` - ***Property: store.type*** The type of the store must be set to "virtuoso" @@ -101,4 +100,3 @@ The credentials of the given user | Required | false | | Valid values | string | | Environment | STORE_VIRTUOSO_PASSWORD | - diff --git a/docs/deploy-and-configure/configuration/explore/dataplatform/index.md b/docs/deploy-and-configure/configuration/explore/dataplatform/index.md index 26645d1b..c4b97f14 100644 --- a/docs/deploy-and-configure/configuration/explore/dataplatform/index.md +++ b/docs/deploy-and-configure/configuration/explore/dataplatform/index.md @@ -2,6 +2,7 @@ tags: - Configuration --- + # Explore backend (DataPlatform) This manual describes how to install and set up eccenca Explore backend (DataPlatform). @@ -11,11 +12,11 @@ To use this manual, system administrators should have knowledge about Linux (Ubu The following subsections describe different configuration topics in detail. Every subsection is presented with a property key overview and a details section with additional explanations: -- [Explore backend general configuration](application-full.md) -- [OAuth specific configuration](application-oauth-full.md) -- Triple Store specific configuration - - [Ontotext GraphDB](application-graphdb-full.md) - - [HTTP](application-http-full.md) - - [In-Memory](application-inmemory-full.md) - - [AWS Neptune](application-neptune-full.md) - - [Openlink Virtuoso](application-virtuoso-full.md) +- [Explore backend general configuration](application-full.md) +- [OAuth specific configuration](application-oauth-full.md) +- Triple Store specific configuration + - [Ontotext GraphDB](application-graphdb-full.md) + - [HTTP](application-http-full.md) + - [In-Memory](application-inmemory-full.md) + - [AWS Neptune](application-neptune-full.md) + - [Openlink Virtuoso](application-virtuoso-full.md) diff --git a/docs/deploy-and-configure/configuration/explore/graph-resource-pattern/index.md b/docs/deploy-and-configure/configuration/explore/graph-resource-pattern/index.md index d7aeeec5..d23d223d 100644 --- a/docs/deploy-and-configure/configuration/explore/graph-resource-pattern/index.md +++ b/docs/deploy-and-configure/configuration/explore/graph-resource-pattern/index.md @@ -1,5 +1,5 @@ --- -#icon: octicons/cross-reference-24 +# icon: octicons/cross-reference-24 tags: - Reference --- @@ -211,5 +211,5 @@ A valid configuration must use a `subjectVarName` called `resource`. This is the This configuration produces the following result, it only shows results where: -- `resource` is of type `http://example.com/vocab/Company` -- a `subResource` exists which is related to `resource` via the `http://example.com/vocab/hasParent` property +- `resource` is of type `http://example.com/vocab/Company` +- a `subResource` exists which is related to `resource` via the `http://example.com/vocab/hasParent` property diff --git a/docs/deploy-and-configure/configuration/explore/index.md b/docs/deploy-and-configure/configuration/explore/index.md index 954b1d8a..6c32a3f5 100644 --- a/docs/deploy-and-configure/configuration/explore/index.md +++ b/docs/deploy-and-configure/configuration/explore/index.md @@ -8,27 +8,26 @@ tags: This page describes how to configure the eccenca Explore component which is done in two parts: -1. The Explore frontend (DataManager) is configured visually through the :eccenca-module-workspace-configuration: _Workspace configuration_ module. -2. [The Explore backend (DataPlatform)](dataplatform/index.md), all details are described on the respective sub-page +1. The Explore frontend (DataManager) is configured visually through the :eccenca-module-workspace-configuration: _Workspace configuration_ module. +2. [The Explore backend (DataPlatform)](dataplatform/index.md), all details are described on the respective sub-page eccenca Explore frontend (DataManager) is a single-page JavaScript application, which means the application consists of a single HTML page which loads all needed web resources in the browser after loading the page itself. In the context of Explore frontend (DataManager), these web resources are: -- The application including its configuration (`app*.js`, `config.js`) -- Styles (`*.css`) -- Web fonts for typography as well as for icons (`*.woff`, `*.ttf`, `*.eot`) -- Images (e.g. logos) (`*.png`, `*.svg`) +- The application including its configuration (`app*.js`, `config.js`) +- Styles (`*.css`) +- Web fonts for typography as well as for icons (`*.woff`, `*.ttf`, `*.eot`) +- Images (e.g. logos) (`*.png`, `*.svg`) Explore frontend (DataManager) communicates with different API endpoints in order to retrieve and manipulate data. The features of Explore frontend (DataManager) include: -- Dataset Manager to create and update datasets and its meta data -- Vocabulary Manager to install and remove Vocabulary descriptions -- Data browser to explore and manage graph-based data -- Taxonomy Editor to manage and create SKOS based taxonomies -- Query editor to query graph-based data via SPARQL queries -- Access control -- Compliance of W3C standards such as [RDF](https://www.w3.org/standards/techs/rdf#w3c_all), [Linked Data](https://www.w3.org/standards/techs/linkeddata#w3c_all) and [SPARQL](https://www.w3.org/standards/techs/sparql#w3c_all) - +- Dataset Manager to create and update datasets and its meta data +- Vocabulary Manager to install and remove Vocabulary descriptions +- Data browser to explore and manage graph-based data +- Taxonomy Editor to manage and create SKOS based taxonomies +- Query editor to query graph-based data via SPARQL queries +- Access control +- Compliance of W3C standards such as [RDF](https://www.w3.org/standards/techs/rdf#w3c_all), [Linked Data](https://www.w3.org/standards/techs/linkeddata#w3c_all) and [SPARQL](https://www.w3.org/standards/techs/sparql#w3c_all) diff --git a/docs/deploy-and-configure/configuration/graphinsights/index.md b/docs/deploy-and-configure/configuration/graphinsights/index.md index adca1410..044555f5 100644 --- a/docs/deploy-and-configure/configuration/graphinsights/index.md +++ b/docs/deploy-and-configure/configuration/graphinsights/index.md @@ -62,8 +62,8 @@ See [Configure OAuth clients (helm)](../../../deploy-and-configure/configuration The configuration mentioned below is rendered with those files, but you usually don't have to touch those: -- `configuration-files/explore-application.yml` for Explore -- `configuration-files/cmem.integration.config.yml` for Graph Insights +- `configuration-files/explore-application.yml` for Explore +- `configuration-files/cmem.integration.config.yml` for Graph Insights For more details please have a look in the helm value file. Every configuration is documented there. @@ -242,8 +242,8 @@ There is the need for a client to authenticate in a browser and a second client For convenience, by default we use the same clients as for the rest of the application: -- Client for browser: cmem -- Client for component communication: cmem-service-account +- Client for browser: cmem +- Client for component communication: cmem-service-account In case you want to have separate clients for production deployments, this chapter is for you. @@ -309,12 +309,12 @@ separate clients are already created when starting the Postgresql container. However, regenerating a new client secret is advisable: -- Select the `eccenca Corporate Memory`-realm -- Select `Clients` -- Select the client `graph-insights-service-account`. -- Select the tab `Credentials` -- Press `Regenerate` -- Then copy the new secret and fill in the values from above. +- Select the `eccenca Corporate Memory`-realm +- Select `Clients` +- Select the client `graph-insights-service-account`. +- Select the tab `Credentials` +- Press `Regenerate` +- Then copy the new secret and fill in the values from above. ![Keycloak client secret regenerate](keycloak-client-secret-regenerate.png) @@ -325,17 +325,17 @@ In an already running deployment you would have to create those clients on your We assume that we name the clients as following: -- Browser client: `graph-insights` -- Inter-component client: `graph-insights-service-account` +- Browser client: `graph-insights` +- Inter-component client: `graph-insights-service-account` #### Set Backchannel logout URL for Graph Insights You would follow the same steps as in [Keycloak Client configuration](../../../deploy-and-configure/configuration/keycloak/index.md) but name them differently, e.g. as above. In addition, one last step is missing: To have the logout working properly you have to add a client Backchannel logout URL for the client meant for browser authentication (`graph-insights`): -- Select the `eccenca Corporate Memory`-realm -- Select `Clients` -- Select the client `graph-insights`. -- Scroll down to this section and add this: `https:///graphinsights/logout/connect/back-channel/keycloak` +- Select the `eccenca Corporate Memory`-realm +- Select `Clients` +- Select the client `graph-insights`. +- Scroll down to this section and add this: `https:///graphinsights/logout/connect/back-channel/keycloak` ![Keycloak backchannel LogExplore select_graphinsights](keycloak-client-backchannel.png) diff --git a/docs/deploy-and-configure/configuration/keycloak/change-passwords-and-keys/index.md b/docs/deploy-and-configure/configuration/keycloak/change-passwords-and-keys/index.md index b5b32d95..8eed7ed7 100644 --- a/docs/deploy-and-configure/configuration/keycloak/change-passwords-and-keys/index.md +++ b/docs/deploy-and-configure/configuration/keycloak/change-passwords-and-keys/index.md @@ -75,4 +75,3 @@ The following make targets will create a database dump, store it in `data/backu ```shell-session make keycloak-backup keycloak-restore ``` - diff --git a/docs/deploy-and-configure/configuration/keycloak/index.md b/docs/deploy-and-configure/configuration/keycloak/index.md index 2eee49ec..97506632 100644 --- a/docs/deploy-and-configure/configuration/keycloak/index.md +++ b/docs/deploy-and-configure/configuration/keycloak/index.md @@ -26,23 +26,23 @@ The screenshots displayed in this documentation were taken from Keycloak v20 usi To create a realm, use the drop down menu for choosing a realm on the left side. -- Create a realm `cmem` +- Create a realm `cmem` - Select **Realm settings** - - **General** tab: - - Change HTML Display name to `Corporate Memory` + - **General** tab: + - Change HTML Display name to `Corporate Memory` - **Themes** tab - - Switch realm's login theme to `eccenca` - - Switch realm's account theme to `eccenca` + - Switch realm's login theme to `eccenca` + - Switch realm's account theme to `eccenca` ## Client configuration Clients are used to link users and groups managed in Keycloak to Corporate Memory. There are two different types of clients used by Corporate Memory: -- The first client is used to authenticate a user for using the web interface (usually named `cmem`). -- The second client is used as a technical user with the command line interface (usually named `cmem-service-account`). -- (optional, when using Graph Insights) similar to the `cmem` client you can configure Graph Insights to use a separate client (usually named `graph-insights`) -- (optional, when using Graph Insights) similar to the `cmem-service-account` client you can configure Graph Insights to communicate with a separate client (usually named `graph-insights-service-account`) +- The first client is used to authenticate a user for using the web interface (usually named `cmem`). +- The second client is used as a technical user with the command line interface (usually named `cmem-service-account`). +- (optional, when using Graph Insights) similar to the `cmem` client you can configure Graph Insights to use a separate client (usually named `graph-insights`) +- (optional, when using Graph Insights) similar to the `cmem-service-account` client you can configure Graph Insights to communicate with a separate client (usually named `graph-insights-service-account`) For Graph Insight please refer the [Graph Insights OAUTH documentation](../../../deploy-and-configure/configuration/graphinsights/index.md#oauth-configuration) Depending on the environment, there might be other use cases, when running background schedules, then a third client, also as technical user, might be useful. @@ -53,11 +53,11 @@ Corporate Memory uses access conditions which are related to users or groups. This is described at [Access Conditions](./../access-conditions/index.md). To use groups from Keycloak in Corporate Memory access conditions, all Keycloak client configurations need to have attached mappers: -- For the web interface client (`cmem`), the user groups need to get attached to the client. +- For the web interface client (`cmem`), the user groups need to get attached to the client. This is done by a **Group Membership** mapper (described below). With this mapper each group of a user is assigned for the authentication process, so Corporate Memory is aware of the user and group IDs for setting up access conditions. -- For the technical account clients (such as `cmem-service-account`), Keycloak does not allow to add groups directly to a client. +- For the technical account clients (such as `cmem-service-account`), Keycloak does not allow to add groups directly to a client. To work around this limitation, we are using **ROLES** instead. By creating a mapper to re-define roles from groups, we allow Corporate Memory to read roles as groups attached to the client token. @@ -73,11 +73,11 @@ To import a pre-configured `cmem` client for using the web interface, follow the ![Dialog import cmem client](import-client-cmem.png) -- Login to Keycloak and select the Corporate Memory realm (`cmem`). -- Download the [client configuration for using the web interface](cmem.json) (`cmem.json`). -- Select **Clients**, then **Import client**. -- **Browse** for the downloaded `cmem.json` and select it. -- **Save** new client. +- Login to Keycloak and select the Corporate Memory realm (`cmem`). +- Download the [client configuration for using the web interface](cmem.json) (`cmem.json`). +- Select **Clients**, then **Import client**. +- **Browse** for the downloaded `cmem.json` and select it. +- **Save** new client. To import a pre-configured `cmem-service-account` client, repeat the process with the [client configuration with credentials for the technical account (`cmem-service-account`)](cmem-service-account.json) (`cmem-service-account.json`). After importing add the `elds-admins` role mapper to the client. See in the manual section of [Add the `cmem-service-account` client](#serviceaccountroles) @@ -89,45 +89,45 @@ This client is intended for the usage with Explore and Build (DataIntegration) ( ![Dialog create cmem client](createClient_1.png){ class="bordered" } -- **Client type**: OpenID Connect -- **Client ID**: i.e. `cmem`, you need to remember this and use this later -- **Name** and **Description**: fill as you like -- Select **Next** -- **Client authentication**: Off -- **Authorization**: Off -- Enable **Standard Flow Enabled** (enables OAuth 2.0 Authorization Code Flow) -- Before v23.1: - - Additionally enable **Implicit Flow Enabled** -- **Save** +- **Client type**: OpenID Connect +- **Client ID**: i.e. `cmem`, you need to remember this and use this later +- **Name** and **Description**: fill as you like +- Select **Next** +- **Client authentication**: Off +- **Authorization**: Off +- Enable **Standard Flow Enabled** (enables OAuth 2.0 Authorization Code Flow) +- Before v23.1: + - Additionally enable **Implicit Flow Enabled** +- **Save** ![Dialog create cmem client](createClient_2.png){ class="bordered" } The dialog above closes and you land on the configuration page of this client: -- **Valid redirect URIs**: Add the correct URL pattern (e.g., wildcard `https://cmem.example.net/*` works) to `Valid Redirect URIs` (`*` for testing purposes can be used as well) and **Save** -- Switch the Tabs to **Client scopes** and click on the first scope (i.e.: `cmem-dedicated`) +- **Valid redirect URIs**: Add the correct URL pattern (e.g., wildcard `https://cmem.example.net/*` works) to `Valid Redirect URIs` (`*` for testing purposes can be used as well) and **Save** +- Switch the Tabs to **Client scopes** and click on the first scope (i.e.: `cmem-dedicated`) ![Dialog select cmem-service-account-dedicated](createClient_11.png){ class="bordered" } ![Dialog create mapper](createClient_4.png){ class="bordered" } ![Dialog create mapper](createClient_5.png){ class="bordered" } -- Click **Configure a new mapper** - - Select Mapper Type **Group Membership** - - **Name** `groups` - - **Token Claim Name** `groups` - - Disable **Full group path** - - Disable **Add to ID token** - - Enable **Add to access token** - - Enable **Add to user info** -- **Save** +- Click **Configure a new mapper** + - Select Mapper Type **Group Membership** + - **Name** `groups` + - **Token Claim Name** `groups` + - Disable **Full group path** + - Disable **Add to ID token** + - Enable **Add to access token** + - Enable **Add to user info** +- **Save** ![Dialog create mapper](createClient_6.png){ class="bordered" } -- In Corporate Memory configuration until v22.2: - - Configure this client ID under `js.config.workspaces.default.authorization.oauth2.clientId` in DataManager's configuration file (Datamanager needs implicit flow) - - Configure this client ID under `oauth.clientId = "cmem"` in DataManager's configuration file (Dataintegration needs standard flow) -- In Corporate Memory configuration from v23.1: - - Configure this client ID in the environments with the name `OAUTH_CLIENT_ID` in `/environments/config.env` (defaults to `cmem` if not set) +- In Corporate Memory configuration until v22.2: + - Configure this client ID under `js.config.workspaces.default.authorization.oauth2.clientId` in DataManager's configuration file (Datamanager needs implicit flow) + - Configure this client ID under `oauth.clientId = "cmem"` in DataManager's configuration file (Dataintegration needs standard flow) +- In Corporate Memory configuration from v23.1: + - Configure this client ID in the environments with the name `OAUTH_CLIENT_ID` in `/environments/config.env` (defaults to `cmem` if not set) #### Add the `cmem-service-account` client @@ -140,75 +140,75 @@ For this, just create a different role name later, and create an access conditio ![Dialog create role](createClient_7_2.png){ class="bordered" } ![Dialog create role](createClient_7_3.png){ class="bordered" } -- **Client type**: OpenID Connect -- **Client ID**: i.e. `cmem-service-account`, you need to remember this and use this later -- **Name** and **Description**: fill as you like -- click **Next** -- **Client authentication**: On -- **Authorization**: Off -- **Authentication flow**: only enable `Service accounts roles`, the rest can be disabled -- **Save** +- **Client type**: OpenID Connect +- **Client ID**: i.e. `cmem-service-account`, you need to remember this and use this later +- **Name** and **Description**: fill as you like +- click **Next** +- **Client authentication**: On +- **Authorization**: Off +- **Authentication flow**: only enable `Service accounts roles`, the rest can be disabled +- **Save** -- Go to **Credentials** and configure **Client Id and Secret**, copy the client secret for later usage +- Go to **Credentials** and configure **Client Id and Secret**, copy the client secret for later usage ![Dialog create role](createClient_7_4.png){ class="bordered" } -- Go to **Roles** and click **Create role** to create the `elds-admins` role +- Go to **Roles** and click **Create role** to create the `elds-admins` role ![Dialog create role](createClient_7.png){ class="bordered" } ![Dialog create role](createClient_8.png){ class="bordered" } -- Click **Action** and select **Add associated roles** +- Click **Action** and select **Add associated roles** ![Dialog create role](createClient_9.png){ class="bordered" } -- Select **Filter by client** from the filter pull-down-menu +- Select **Filter by client** from the filter pull-down-menu ![Dialog create role](createClient_10.png){ class="bordered" } -- In this dialog select the client by name which you are currently configuring (here `cmem-service-account`) and then **Assign** +- In this dialog select the client by name which you are currently configuring (here `cmem-service-account`) and then **Assign** ![Dialog create role](createClient_10_1.png){ class="bordered" } -- Go back to **Client details** e.g., by using the top navigation -- In the **Roles** tab you now see your created role here +- Go back to **Client details** e.g., by using the top navigation +- In the **Roles** tab you now see your created role here ![Dialog create role](createClient_10_2.png){ class="bordered" } -- Switch the Tabs to **Client scopes** and click on the first scope (i.e.: `cmem-service-account-dedicated`) +- Switch the Tabs to **Client scopes** and click on the first scope (i.e.: `cmem-service-account-dedicated`) ![Dialog create mapper](createClient_11.png){ class="bordered" } -- select **Add mapper** -> **By configuration** +- select **Add mapper** -> **By configuration** ![Dialog create mapper](createClient_13_1.png){ class="bordered" } -- select Mapper Type `User Client Role` - - **Name** `roles` - - **Client ID** select the client you are currently configuring from the pull-down-menu (here `cmem-service-account`) - - Enable **Multivalued** - - **Token Claim Name** `groups` - - Enable **Add to ID token** - - Enable **Add to access token** - - Enable **Add to user info** -- **Save** +- select Mapper Type `User Client Role` + - **Name** `roles` + - **Client ID** select the client you are currently configuring from the pull-down-menu (here `cmem-service-account`) + - Enable **Multivalued** + - **Token Claim Name** `groups` + - Enable **Add to ID token** + - Enable **Add to access token** + - Enable **Add to user info** +- **Save** ![Dialog create mapper](createClient_13.png){ class="bordered" } ![Dialog create mapper](createClient_14.png){ class="bordered" } -- After **Save** go back to **Client details** -- Go to **Service account roles** tab -- Select the link in the center **To manage detail and group mappings, click on the username service-account-YOUR_CLIENT_ID** +- After **Save** go back to **Client details** +- Go to **Service account roles** tab +- Select the link in the center **To manage detail and group mappings, click on the username service-account-YOUR_CLIENT_ID** ![Dialog add role to client](createClient_15.png){ class="bordered" } -- Go to tab **Role mapping** and select **Assign role** +- Go to tab **Role mapping** and select **Assign role** ![Dialog add role to client](createClient_16.png){ class="bordered" } -- Change the filter to **Filter by clients** and select the new Client ID, i.e `cmem-service-account` -- Click **Assign** +- Change the filter to **Filter by clients** and select the new Client ID, i.e `cmem-service-account` +- Click **Assign** ![Dialog add role to client](createClient_16_2.png){ class="bordered" } ![Dialog add role to client](createClient_16_1.png){ class="bordered" } @@ -216,8 +216,8 @@ For this, just create a different role name later, and create an access conditio ## Corporate Memory configuration after setting up clients -- If **Build (DataIntegration)** schedulers are required, configure this client id and secret under the properties `workbench.superuser.client` and `workbench.superuser.clientSecret` in Build (DataIntegration)'s configuration file or -- in `docker compose`-orchestration you can edit this in the environment as: +- If **Build (DataIntegration)** schedulers are required, configure this client id and secret under the properties `workbench.superuser.client` and `workbench.superuser.clientSecret` in Build (DataIntegration)'s configuration file or +- in `docker compose`-orchestration you can edit this in the environment as: ``` bash CMEM_SERVICE_ACCOUNT_CLIENT_ID=cmem-service-account @@ -226,27 +226,27 @@ For this, just create a different role name later, and create an access conditio DATAINTEGRATION_CMEM_SERVICE_CLIENT_SECRET=YourSecret ``` -- in helm this value is defined by: +- in helm this value is defined by: ``` yaml DATAINTEGRATION_CMEM_SERVICE_CLIENT_SECRET: {{ .Values.global.cmemClientSecret }} DATAINTEGRATION_CMEM_SERVICE_CLIENT: {{ .Values.global.cmemClientId }} ``` -- For **cmemc** you can configure this with `OAUTH_CLIENT_ID` and `OAUTH_CLIENT_SECRET`. +- For **cmemc** you can configure this with `OAUTH_CLIENT_ID` and `OAUTH_CLIENT_SECRET`. ### Groups configuration -- Go to **Groups** and add the following groups: - - `elds-admins` - - These groups are used only to assign them to user accounts (clients have roles-to-group mappers). - - Any groups provided by your user management system (e.g. LDAP) that must be recognized/mapped by Keycloak - - Corporate Memory does not come with any other groups. Those are optional and can be defined here. +- Go to **Groups** and add the following groups: + - `elds-admins` + - These groups are used only to assign them to user accounts (clients have roles-to-group mappers). + - Any groups provided by your user management system (e.g. LDAP) that must be recognized/mapped by Keycloak + - Corporate Memory does not come with any other groups. Those are optional and can be defined here. ### Users configuration -- This applies to the [Docker Orchestration](./../docker-orchestration/index.md), for other setups consult the [Keycloak manual](https://www.keycloak.org/docs/latest/server_admin/). -- Go to `Users` -- Add the following users and assign their groups respectively (for each user go to credentials, add password and disable `Temporary`) - - `admin:admin` - - groups: `elds-admins` +- This applies to the [Docker Orchestration](./../docker-orchestration/index.md), for other setups consult the [Keycloak manual](https://www.keycloak.org/docs/latest/server_admin/). +- Go to `Users` +- Add the following users and assign their groups respectively (for each user go to credentials, add password and disable `Temporary`) + - `admin:admin` + - groups: `elds-admins` diff --git a/docs/deploy-and-configure/configuration/keycloak/using-external-keycloak/index.md b/docs/deploy-and-configure/configuration/keycloak/using-external-keycloak/index.md index fb3e9888..9dc58358 100644 --- a/docs/deploy-and-configure/configuration/keycloak/using-external-keycloak/index.md +++ b/docs/deploy-and-configure/configuration/keycloak/using-external-keycloak/index.md @@ -19,9 +19,9 @@ For this scenario, this page provides additional configuration requirements. Depending on your infrastructure around Corporate Memory, you need to change some provisioned HTTP header on the following services: - Headers for Keycloak URLs: - - `Access-Control-Allow-Origin: https://cmem.example.com` + - `Access-Control-Allow-Origin: https://cmem.example.com` - Headers for Corporate Memory URLs: - - `Access-Control-Allow-Origin: *` + - `Access-Control-Allow-Origin: *` For example, if you are using our helm charts, adapt the followin ingress annotations: @@ -37,7 +37,6 @@ For example, if you are using our helm charts, adapt the followin ingress annota nginx.ingress.kubernetes.io/cors-allow-origin: "*" ``` - ### Keycloak You have to allow the Corporate Memory domain in the Keycloak settings. @@ -122,4 +121,3 @@ In either way you can configure the base realm path in the global value section: # keycloak oauth client id (used for DataPlatform connection and DataIntegration cmem service client) oauthClientId: cmem ``` - diff --git a/docs/deploy-and-configure/configuration/label-resolution-and-full-text-search/index.md b/docs/deploy-and-configure/configuration/label-resolution-and-full-text-search/index.md index e36988fe..9536ffe4 100644 --- a/docs/deploy-and-configure/configuration/label-resolution-and-full-text-search/index.md +++ b/docs/deploy-and-configure/configuration/label-resolution-and-full-text-search/index.md @@ -9,9 +9,9 @@ This resolution and, by extension, the full text search is configurable for diff eccenca Explore backend (DataPlatform) offers three configuration options: -- `labelProperties` (line 2)  -- `languagePreferences` (line 5) and  -- `languagePreferencesAnyLangFallback` (line 8). +- `labelProperties` (line 2)  +- `languagePreferences` (line 5) and  +- `languagePreferencesAnyLangFallback` (line 8). ``` yaml linenums="1" proxy: @@ -28,8 +28,8 @@ These properties define not only which properties and languages should be consid The retrieval process can be simplified to the following procedure: -- First, when determining the label for a resource, the language is evaluated, then the property is considered. -- Consequently, for a resource in the default case: +- First, when determining the label for a resource, the language is evaluated, then the property is considered. +- Consequently, for a resource in the default case: 1. An english value for `rdfs:label` is searched. 2. A literal of the property `rdfs:label` without a language tag is searched (which is why there is an entry `""`). 3. An english value of `skos:prefLabel` is searched. @@ -50,12 +50,12 @@ How labels are resolved is best explained using these default settings and some :Resource4 rdfs:label "Another Label for Hanover"@en ``` -- For `:Resource1` the label will be `Leipzig` as the english `rdfs:label` will be retrieved. -- For `:Resource2` the label cannot be retrieved from the Knowledge Graph since no known property is used. Hence the fallback. -- For `:Resource3` the label will be retrieved as `Stuttgart`, if the `languagePreferencesAnyLangFallback` is `true`. - - While there is a well-known property used, none of the used languages match. Using the fallback, the alphabetically first match is retrieved in this case. -- For `:Resource4` multiple label candidates could be determined. - - In this case, `Another Label for Hanover` is retrieved as it is the first value in the alphanumerical comparison. +- For `:Resource1` the label will be `Leipzig` as the english `rdfs:label` will be retrieved. +- For `:Resource2` the label cannot be retrieved from the Knowledge Graph since no known property is used. Hence the fallback. +- For `:Resource3` the label will be retrieved as `Stuttgart`, if the `languagePreferencesAnyLangFallback` is `true`. + - While there is a well-known property used, none of the used languages match. Using the fallback, the alphabetically first match is retrieved in this case. +- For `:Resource4` multiple label candidates could be determined. + - In this case, `Another Label for Hanover` is retrieved as it is the first value in the alphanumerical comparison. ## Client API diff --git a/docs/deploy-and-configure/configuration/production-ready-settings/index.md b/docs/deploy-and-configure/configuration/production-ready-settings/index.md index 4cfe4945..cc694218 100644 --- a/docs/deploy-and-configure/configuration/production-ready-settings/index.md +++ b/docs/deploy-and-configure/configuration/production-ready-settings/index.md @@ -29,6 +29,7 @@ For Explore backend (DataPlatform) you set this in `application.yml` or as envir ```yaml deploy.post-logout-redirect-uri: "${DEPLOY_BASE_URL}" ``` + ```bash DEPLOY_POST_LOGOUT_REDIRECT_URI=${DEPLOY_BASE_URL} ``` @@ -50,7 +51,6 @@ oauth.logoutRedirectUrl = ${OAUTH_LOGOUT_REDIRECT_URL} oauth.idToken = true ``` - ## Password Policies If you create users in Keycloak, make sure these users have strong passwords. @@ -93,7 +93,6 @@ play.http.session.secure = ${DATAINTEGRATION_SECURE_COOKIE} In the [Play documentation](https://www.playframework.com/documentation/2.8.x/SettingsSession), you can find further information, i.e. also setting `sameSite = "lax"`or `strict`. By default Build (DataIntegration) sets this to `lax` - ## CORS Settings ### Explore backend (DataPlatform) @@ -145,4 +144,3 @@ cors.config.allowOrigins = "*" # If set to true, allowOrigins must not have '*' configured. cors.config.allowCredentials = false ``` - diff --git a/docs/deploy-and-configure/configuration/quad-store-configuration/index.md b/docs/deploy-and-configure/configuration/quad-store-configuration/index.md index d256176b..18171cf2 100644 --- a/docs/deploy-and-configure/configuration/quad-store-configuration/index.md +++ b/docs/deploy-and-configure/configuration/quad-store-configuration/index.md @@ -13,7 +13,7 @@ Ensure to add the suggested parameters to the corresponding subsections of the c ### Compatibility -- **Virtuoso 7.2.4.2** - Explore is compatible with [Virtuoso 7.2.4.2](https://github.com/openlink/virtuoso-opensource/releases/tag/v7.2.4.2). Compatibility with other versions is not guaranteed. +- **Virtuoso 7.2.4.2** - Explore is compatible with [Virtuoso 7.2.4.2](https://github.com/openlink/virtuoso-opensource/releases/tag/v7.2.4.2). Compatibility with other versions is not guaranteed. ### Configuration @@ -66,36 +66,36 @@ This section only covers limitations and options which have a direct impact on t ### Compatibility -- **Stardog 7.2.1** - Explore is compatible with [Stardog version 7.1.1](http://docs.stardog.com/). Compatibility with newer versions is not guaranteed. -- Stardog 6.2.3 (deprecated) - Explore is compatible with [Stardog version 6.2.3](https://www.stardog.com/docs/6.2.3/). +- **Stardog 7.2.1** - Explore is compatible with [Stardog version 7.1.1](http://docs.stardog.com/). Compatibility with newer versions is not guaranteed. +- Stardog 6.2.3 (deprecated) - Explore is compatible with [Stardog version 6.2.3](https://www.stardog.com/docs/6.2.3/). !!! note Support for 6.2.3 is deprecated and will be removed in later Explore releases. ### Configuration -- **Search enabled**\ +- **Search enabled**\ Explore relies on the Stardog Semantic Search, which has to be enabled by setting: - - `search.enabled=true`\ + - `search.enabled=true`\ You can set this property using either Stardog Studio or the `stardog-admin` commands.\ Refer to the [Stardog documentation](https://www.stardog.com/docs/#_configuration_options) for more detailed information. -- **Server side named graph security**\ +- **Server side named graph security**\ If the `PROVISIONED` access control strategy is used for the configured endpoint, you have to set the property `security.named.graphs=true` for the configured database as explained in the [Stardog documentation](http://docs.stardog.com/#_named_graph_security). Additionally, the following properties are required: - - `password.length.max`: For the provisioned mode to work properly. This property should have a value of at least 64. - - `password.regex`: The default value configured in Stardog is not compatible with the passwords generated by Explore. The regex should be `[\\w+\\/=]+` + - `password.length.max`: For the provisioned mode to work properly. This property should have a value of at least 64. + - `password.regex`: The default value configured in Stardog is not compatible with the passwords generated by Explore. The regex should be `[\\w+\\/=]+` -- **SSL support**\ +- **SSL support**\ Mandatory configuration if `sparqlEndpoints.stardog[i].sslEnabled=true`. The server must have a valid certificate which must be trusted by the system where Explore runs. In this case, the `sparqlEndpoints.stardog[i].port` property must point to the SSL port (which default value is `5821`).\ Consult the [Configuring Stardog to use SSL](https://www.stardog.com/docs/#_configuring_stardog_to_use_ssl) section of Stardog's manual for more information on the topic. -- **Query timeout override**\ +- **Query timeout override**\ In order to allow Explore to override the query timeout for individual queries, you have to ensure that the property `query.timeout.override.enabled` for the database is set to `true` (which is the default).\ Consult the [Configuring Query Management](https://www.stardog.com/docs/#_configuring_query_management) section of Stardog's manual for further information. ### Limitations -- **Quad format upload**\ +- **Quad format upload**\ The Graph Store Protocol implementation for Stardog does not support uploading of RDF quad data (TriG, N-Quads). -- **Initial connection**\ +- **Initial connection**\ The first request to Explore can take several seconds due to connection startup to the Stardog server. ### Caveats @@ -166,7 +166,7 @@ This section covers only limitations and options which have a direct impact on t ### Compatibility -- GraphDB 9.2.0 - Explore is compatible with [GraphDB version 9.2.0](http://graphdb.ontotext.com/documentation/standard/release-notes.html#graphdb-9-2-0). Compatibility with newer versions is not guaranteed. +- GraphDB 9.2.0 - Explore is compatible with [GraphDB version 9.2.0](http://graphdb.ontotext.com/documentation/standard/release-notes.html#graphdb-9-2-0). Compatibility with newer versions is not guaranteed. ### Configuration diff --git a/docs/deploy-and-configure/configuration/reverse-proxy/index.md b/docs/deploy-and-configure/configuration/reverse-proxy/index.md index 527e520c..13b040b6 100644 --- a/docs/deploy-and-configure/configuration/reverse-proxy/index.md +++ b/docs/deploy-and-configure/configuration/reverse-proxy/index.md @@ -8,9 +8,9 @@ A reverse proxy forwards all requests from the users to the called service and r Reverse proxy is a necessary component in the Corporate Memory deployment. It enables you to: -- define routes for all the components within one domain name -- expose only ports 80 and 443 to the outside network, all other communication would be performed in the internal network -- ease configuration and management of the SSL certificates +- define routes for all the components within one domain name +- expose only ports 80 and 443 to the outside network, all other communication would be performed in the internal network +- ease configuration and management of the SSL certificates This also enables you to activate the [Linked Data delivery mode](#linked-data-delivery-mode) of Explore. The Linked Data delivery mode is able to serve Linked Data that uses the same namespace as the configured domain name as resolvable URIs including content negotiation. @@ -80,9 +80,9 @@ The Linked Data delivery mode is able to serve data that uses the same namespace Therefore you can use the following template (e.g.: ): --  (DataPlatform) --  (a custom vocabulary) -- with HTTPS enforcement (recommended) +-  (DataPlatform) +-  (a custom vocabulary) +- with HTTPS enforcement (recommended) apache sample config for linked data delivery diff --git a/docs/deploy-and-configure/index.md b/docs/deploy-and-configure/index.md index 9164114b..395e9606 100644 --- a/docs/deploy-and-configure/index.md +++ b/docs/deploy-and-configure/index.md @@ -11,29 +11,28 @@ Deploy and configure eccenca Corporate Memory in your own environment.
-- :eccenca-application-homepage: [System Architecture](system-architecture/index.md) +- :eccenca-application-homepage: [System Architecture](system-architecture/index.md) --- This page describes the overall system architecture of eccenca Corporate Memory and its components. -- :material-exclamation-thick: [Requirements](requirements/index.md) +- :material-exclamation-thick: [Requirements](requirements/index.md) --- This page lists software and hardware requirements for eccenca Corporate Memory deployments. -- :material-download-circle-outline: [Installation](installation/index.md) +- :material-download-circle-outline: [Installation](installation/index.md) --- These pages describe proven deployment scenarios for eccenca Corporate Memory. -- :material-cog-outline: [Configuration](configuration/index.md) +- :material-cog-outline: [Configuration](configuration/index.md) --- These pages describe specific topics on how to configure eccenca Corporate Memory.
- diff --git a/docs/deploy-and-configure/installation/migrating-stores/index.md b/docs/deploy-and-configure/installation/migrating-stores/index.md index 3443685a..a3930cf2 100644 --- a/docs/deploy-and-configure/installation/migrating-stores/index.md +++ b/docs/deploy-and-configure/installation/migrating-stores/index.md @@ -28,7 +28,7 @@ icon: material/database - best practice: - run all (SELECT) queries in the query catalog and compare results (e.g. with `cmemc`) - - theoretically this could also be applied to INSERT queries (by re-writing into SELECTS in case you want / need to omit altering your graphs) + - theoretically this could also be applied to INSERT queries (by re-writing into SELECTS in case you want / need to omit altering your graphs) - count all triples in all graphs on both instances before/after export/import (`cmemc graph count --all`) ## Optimizing Your Setup diff --git a/docs/deploy-and-configure/installation/scenario-single-node-cloud-installation/index.md b/docs/deploy-and-configure/installation/scenario-single-node-cloud-installation/index.md index fae8e80f..f35d7de3 100644 --- a/docs/deploy-and-configure/installation/scenario-single-node-cloud-installation/index.md +++ b/docs/deploy-and-configure/installation/scenario-single-node-cloud-installation/index.md @@ -9,11 +9,11 @@ This page describes a docker-compose based orchestration running on a server ins ## Requirements -- ssh access to a server instance (Debian 11) with a public IP address -- A resolvable domain name to this server -- Terminal with ssh client installed locally -- An eccenca partner account for the docker registry as well as the release artifact area -- A GraphDB license ([free](https://www.ontotext.com/products/graphdb/) or commercial) +- ssh access to a server instance (Debian 11) with a public IP address +- A resolvable domain name to this server +- Terminal with ssh client installed locally +- An eccenca partner account for the docker registry as well as the release artifact area +- A GraphDB license ([free](https://www.ontotext.com/products/graphdb/) or commercial) !!! Info make - do not use version 4.4.1 [→](https://savannah.gnu.org/bugs/?63650) @@ -165,7 +165,6 @@ systemctl start cmem-orchestration If you want to have Graph Insights enabled as well, first you need a license. Then you can simiply run this: - ``` shell mkdir licenses ln -s your-license-file.lic graphinsights.lic @@ -190,7 +189,6 @@ You can now proceed to the :material-arrow-right: [Getting Started](../../../ge Do not forget to change the passwords of your deployment, especially if it is available from the public internet. For this, take a look at [Change Passwords and Keys](../../configuration/keycloak/change-passwords-and-keys/index.md). - ```bash cp /opt/cmem-orchestration/conf/systemd/cmem-orchestration.service \ /etc/systemd/system @@ -233,4 +231,5 @@ zip -r data/backups/2024-07-26_14-15.zip data/backups/keycloak/2024-07-26_14-15. ln -sf 2024-07-26_14-15.zip data/backups/latest.zip ``` + The full backup is now at `data/backups/latest.zip`. diff --git a/docs/deploy-and-configure/requirements/graph-insights-sizing.md b/docs/deploy-and-configure/requirements/graph-insights-sizing.md index 5b53e1be..e1b06def 100644 --- a/docs/deploy-and-configure/requirements/graph-insights-sizing.md +++ b/docs/deploy-and-configure/requirements/graph-insights-sizing.md @@ -34,10 +34,10 @@ Since Graph Insights uses caching for performance reasons the latter will increa Disk space consumption can be estimated in close relation to working memory usage: -- On average, **~30 MB per 1 million triples** are required both in memory and on disk. -- In **managed mode**, Graph Insights maintains two index directories (to support hot-swapping during reindexing). +- On average, **~30 MB per 1 million triples** are required both in memory and on disk. +- In **managed mode**, Graph Insights maintains two index directories (to support hot-swapping during reindexing). In this case, disk usage is roughly **~60 MB per 1 million triples**. -- Actual disk usage may vary depending on dataset characteristics (e.g., schema, dictionary compression ratio, and indexing options). +- Actual disk usage may vary depending on dataset characteristics (e.g., schema, dictionary compression ratio, and indexing options). For practical planning, disk space can be roughly approximated by scaling the memory requirements listed in the table above with the adjustment for managed mode if applicable. @@ -79,5 +79,5 @@ The deployed exploration operations in our benchmark consisted of multiple queri We ran 847 exploration queries, selected to be representative of the dataset (covering different group node sizes and varying numbers of connected resources). The experiments were conducted across a variety of datasets up to 250M triples: -- **Maximum execution time:** ~10s for 1-2 threads, improved to ~5s with 3 threads, and ~4.4s with 4 or more threads. -- **75th percentile:** ~2.6-2.8s with 1-2 threads, improved to ~1.3s with 4 threads, and ~1s with 6+ threads.- **Beyond 4 threads:** no further improvement in maximum times was observed, though 75th percentile times continued to improve slightly. +- **Maximum execution time:** ~10s for 1-2 threads, improved to ~5s with 3 threads, and ~4.4s with 4 or more threads. +- **75th percentile:** ~2.6-2.8s with 1-2 threads, improved to ~1.3s with 4 threads, and ~1s with 6+ threads.- **Beyond 4 threads:** no further improvement in maximum times was observed, though 75th percentile times continued to improve slightly. diff --git a/docs/deploy-and-configure/requirements/index.md b/docs/deploy-and-configure/requirements/index.md index f9257285..b90726e1 100644 --- a/docs/deploy-and-configure/requirements/index.md +++ b/docs/deploy-and-configure/requirements/index.md @@ -12,21 +12,21 @@ For a general overview of a deployment setup please refer to the [System Archit A minimal single-node deployment for testing/evaluation purposes means: -- no memory consuming linking and transformation workflows, -- nearly no concurrent users. +- no memory consuming linking and transformation workflows, +- nearly no concurrent users. Depending on how much RAM is dedicated to the triple store, Knowledge Graphs up to several million triples can be built and served. -- Operating System / Hardware - - Bare metal server or VM with Debian based linux OS (see [Installation](./../installation/index.md) for details) - - 16 GB RAM - - 100 GB free disk space (10 GB for docker images + data + logs over time) - - docker and docker compose (we deliver an orchestration including all needed components) +- Operating System / Hardware + - Bare metal server or VM with Debian based linux OS (see [Installation](./../installation/index.md) for details) + - 16 GB RAM + - 100 GB free disk space (10 GB for docker images + data + logs over time) + - docker and docker compose (we deliver an orchestration including all needed components) For an example of a single-node installation refer to the following scenarios: -- [Scenario: Local Installation](../installation/scenario-local-installation/index.md) -- [Scenario: Single Node Cloud Installation](../installation/scenario-single-node-cloud-installation/index.md) +- [Scenario: Local Installation](../installation/scenario-local-installation/index.md) +- [Scenario: Single Node Cloud Installation](../installation/scenario-single-node-cloud-installation/index.md) ## Typical Setup @@ -50,16 +50,15 @@ For GraphDB always also have a look at [GraphDB recommendations](https://graphdb but on the sum of triples in all Graph Insights Snapshots. Have a look at the cmemc command group [graph insights](../../automate/cmemc-command-line-interface/command-reference/graph/insights/index.md) to manage existing snapshots. - ## Clients ### Browser / Web Client We support all (LTS/ESR) versions of the below listed browsers that are actively supported be the respective publishers: -- Microsoft Edge > v88.0 -- Google Chrome or Chromium > v92.0 -- Firefox > v78.0 +- Microsoft Edge > v88.0 +- Google Chrome or Chromium > v92.0 +- Firefox > v78.0 !!! note diff --git a/docs/deploy-and-configure/system-architecture/index.md b/docs/deploy-and-configure/system-architecture/index.md index ecbb656e..a44006f6 100644 --- a/docs/deploy-and-configure/system-architecture/index.md +++ b/docs/deploy-and-configure/system-architecture/index.md @@ -15,13 +15,13 @@ This page describes the overall system architecture of eccenca Corporate Memory eccenca Corporate Memory consists of three core components: -- [eccenca Build](../configuration/dataintegration/index.md) -- [eccenca Explore](../configuration/explore/index.md), -- [cmemc (Corporate Memory Control)](../../automate/cmemc-command-line-interface/index.md), +- [eccenca Build](../configuration/dataintegration/index.md) +- [eccenca Explore](../configuration/explore/index.md), +- [cmemc (Corporate Memory Control)](../../automate/cmemc-command-line-interface/index.md), and one optional component: -- [eccenca Graph Insights](../../explore-and-author/graph-exploration/graph-insights/index.md). +- [eccenca Graph Insights](../../explore-and-author/graph-exploration/graph-insights/index.md). _Build_ is the Corporate Memory component which enables integration of datasets into a single consistent knowledge graph. Datasets in their original format are mapped to RDF schemata and then linked to and persisted into a knowledge graph. diff --git a/docs/develop/accessing-graphs-with-java-applications/index.md b/docs/develop/accessing-graphs-with-java-applications/index.md index 7031c0fe..035f6c4a 100644 --- a/docs/develop/accessing-graphs-with-java-applications/index.md +++ b/docs/develop/accessing-graphs-with-java-applications/index.md @@ -95,4 +95,3 @@ public class HTTPClient { } } ``` - diff --git a/docs/develop/cmemc-scripts/index.md b/docs/develop/cmemc-scripts/index.md index 16d941ff..034396a9 100644 --- a/docs/develop/cmemc-scripts/index.md +++ b/docs/develop/cmemc-scripts/index.md @@ -25,7 +25,6 @@ cmemc is published as an Apache 2 licensed open source python package at [pypi.o pip install cmem-cmemc ``` - ## Configure a connection Assuming you have already [configured your cmemc connection](../../automate/cmemc-command-line-interface/configuration/file-based-configuration/index.md), using it in Python scripts is quite easy. @@ -159,4 +158,3 @@ Workflow 'only-input-replaceable' in 'io' Workflow 'only-output' in 'io' Workflow 'only-output-replaceable' in 'io' ``` - diff --git a/docs/develop/cmempy-python-api/index.md b/docs/develop/cmempy-python-api/index.md index 7282dca7..d412ec56 100644 --- a/docs/develop/cmempy-python-api/index.md +++ b/docs/develop/cmempy-python-api/index.md @@ -30,7 +30,7 @@ The following table lists all processed environment variables: | Variable | Description | Default Value | | ------------------- | ----------------------------------------------------- | ------------------------------------------------------------ | -| CMEM_BASE_URI | Base URL of your Corporate Memory | http://docker.localhost | +| CMEM_BASE_URI | Base URL of your Corporate Memory | | | DI_API_ENDPOINT | Build (Data Integration) API endpoint | CMEM_BASE_URI/dataintegration | | DP_API_ENDPOINT | Explore backend API endpoint | CMEM_BASE_URI/dataplatform | | OAUTH_TOKEN_URI | OAuth 2.0 Token endpoint | CMEM_BASE_URI/auth/realms/cmem/protocol/openid-connect/token | diff --git a/docs/develop/dataintegration-apis/index.md b/docs/develop/dataintegration-apis/index.md index d0e6d2cd..5c7f468c 100644 --- a/docs/develop/dataintegration-apis/index.md +++ b/docs/develop/dataintegration-apis/index.md @@ -30,4 +30,3 @@ Dependent on the specific API, eccenca Build (DataIntegration) works with the fo | application/xml | [XML Media Types](https://tools.ietf.org/html/rfc7303) | | application/n-triples | [RDF 1.1 N-Triples - A line-based syntax for an RDF graph](https://www.w3.org/TR/n-triples/) | | application/problem+json | [Problem Details for HTTP APIs](https://tools.ietf.org/html/rfc7807) | - diff --git a/docs/develop/dataplatform-apis/index.md b/docs/develop/dataplatform-apis/index.md index 70defe02..e4f18d0e 100644 --- a/docs/develop/dataplatform-apis/index.md +++ b/docs/develop/dataplatform-apis/index.md @@ -94,8 +94,8 @@ Furthermore, it says: > > The FROM and FROM NAMED keywords allow a query to specify an RDF dataset by reference; they indicate that the dataset should include graphs that are obtained from representations of the resources identified by the given IRIs (i.e. the absolute form of the given IRI references). The dataset resulting from a number of FROM and FROM NAMED clauses is: > -> - a default graph consisting of the RDF merge of the graphs referred to in the FROM clauses, and -> - a set of (IRI, graph) pairs, one from each FROM NAMED clause. +> - a default graph consisting of the RDF merge of the graphs referred to in the FROM clauses, and +> - a set of (IRI, graph) pairs, one from each FROM NAMED clause. > > If there is no FROM clause, but there is one or more FROM NAMED clauses, then the dataset includes an empty graph for the default graph. @@ -105,28 +105,28 @@ For this reason, Explore backend **does not allow the manipulation of the servic To enforce this policy, the following restriction applies to incoming [SPARQL 1.1](https://www.w3.org/TR/sparql11-update/) Update queries: -- Update queries (INSERT DATA, DELETE DATA and DELETE/INSERT) targeted against the service's default graph will not be accepted by returning an HTTP 400 Bad Request status code. +- Update queries (INSERT DATA, DELETE DATA and DELETE/INSERT) targeted against the service's default graph will not be accepted by returning an HTTP 400 Bad Request status code. ### Default RDF dataset The interpretation of the RDF dataset of a query differs between various SPARQL service implementations. In the case a query declares no RDF dataset, Explore backend uses the following default RDF dataset declaration to provide a uniform behavior for all supported SPARQL services: -- The default graph is the union ([RDF Merge graph](https://www.w3.org/TR/sparql11-query/#sparqlDataset)) of all named graphs the user is allowed to access. -- The set of named graphs contains all named graphs the user is allowed to access. +- The default graph is the union ([RDF Merge graph](https://www.w3.org/TR/sparql11-query/#sparqlDataset)) of all named graphs the user is allowed to access. +- The set of named graphs contains all named graphs the user is allowed to access. ## HTTP error responses The default format for HTTP error responses is compliant with [RFC 7807 Problem Details for HTTP APIs](https://tools.ietf.org/html/rfc7807). An HTTP error response contains a JSON object that provides at least two fields: -- `title`: A short, human-readable summary of the problem type. -- `detail`: A human-readable explanation specific to this occurrence of the problem. +- `title`: A short, human-readable summary of the problem type. +- `detail`: A human-readable explanation specific to this occurrence of the problem. The following optional non-standard fields may also be set: -- `status`: The HTTP status code for this occurrence of the problem. -- `cause`: The cause for this occurrence of the problem. It contains at least the same elements as specified previously, such as `title` and `detail`. +- `status`: The HTTP status code for this occurrence of the problem. +- `cause`: The cause for this occurrence of the problem. It contains at least the same elements as specified previously, such as `title` and `detail`. The following example shows an HTTP response containing JSON problem details using the `application/problem+json` media type: diff --git a/docs/develop/index.md b/docs/develop/index.md index 40561f98..c31d88f0 100644 --- a/docs/develop/index.md +++ b/docs/develop/index.md @@ -9,19 +9,19 @@ API documentation and programming recipes.
-- :material-language-java: Java +- :material-language-java: Java --- [Accessing Graphs with Java Applications](accessing-graphs-with-java-applications/index.md) covers how to connect to Corporate Memory using a Java program. -- :material-language-python: Python +- :material-language-python: Python --- For Python developers, we offer a [Plugin SDK](python-plugins/index.md) as well as an API for accessing and manipulating Corporate Memory Instances ([cmem-cmempy](cmempy-python-api/index.md)). -- :material-api: OpenAPI specification +- :material-api: OpenAPI specification --- diff --git a/docs/develop/python-plugins/development/index.md b/docs/develop/python-plugins/development/index.md index 0a3bd58e..0c9c13ad 100644 --- a/docs/develop/python-plugins/development/index.md +++ b/docs/develop/python-plugins/development/index.md @@ -32,9 +32,9 @@ A workflow plugin may accept an arbitrary list of inputs and optionally returns The lifecycle of a workflow plugin is as follows: -- The plugin will be instantiated once the workflow execution reaches the respective plugin. -- The `execute` function is called and gets the results of the ingoing operators as input. -- The output is forwarded to the next operator. +- The plugin will be instantiated once the workflow execution reaches the respective plugin. +- The `execute` function is called and gets the results of the ingoing operators as input. +- The output is forwarded to the next operator. The following depiction shows a task of the plugin **My Workflow Plugin**. The task has two connected incoming tasks and one connected outgoing task. @@ -100,13 +100,13 @@ The `PluginParameter` class can be instantiated multiple times within a `@Plugin The `PluginParameter` has several parameters that can be specified when initializing an instance: -- `name`: The name of the parameter. This is a required parameter and must be specified. -- `label`: A visible label of the parameter. This is an optional parameter and can be left blank. If left blank, the name of the parameter will be used as the label. -- `description`: A visible description of the parameter. This is an optional parameter and can be left blank. -- `param_type`: Optionally overrides the parameter type. Usually, this does not have to be set manually as it will be inferred from the plugin automatically. -- `default_value`: The parameter default value (optional). If not specified, it will be inferred from the plugin automatically. -- `advanced`: A boolean flag indicating whether or not this is an advanced parameter that can only be changed in the advanced section. This is an optional parameter and defaults to False. -- `visible`: A boolean flag indicating whether or not the parameter will be displayed to the user in the UI. This is an optional parameter and defaults to True. +- `name`: The name of the parameter. This is a required parameter and must be specified. +- `label`: A visible label of the parameter. This is an optional parameter and can be left blank. If left blank, the name of the parameter will be used as the label. +- `description`: A visible description of the parameter. This is an optional parameter and can be left blank. +- `param_type`: Optionally overrides the parameter type. Usually, this does not have to be set manually as it will be inferred from the plugin automatically. +- `default_value`: The parameter default value (optional). If not specified, it will be inferred from the plugin automatically. +- `advanced`: A boolean flag indicating whether or not this is an advanced parameter that can only be changed in the advanced section. This is an optional parameter and defaults to False. +- `visible`: A boolean flag indicating whether or not the parameter will be displayed to the user in the UI. This is an optional parameter and defaults to True. ## Parameter Type @@ -278,18 +278,18 @@ The `EnumParameterType`is an example of a parameter type that uses auto-completi `autocomplete()` takes in three parameters: `query_terms`, `depend_on_parameter_values`, and `context`. It returns a list of `Autocompletion` objects, which represent the possible auto-completion results. -- The `query_terms` parameter is a list of lower case conjunctive search terms. These are the search terms that the user has entered, and the `auto-completion()` will attempt to find results that match all of them. +- The `query_terms` parameter is a list of lower case conjunctive search terms. These are the search terms that the user has entered, and the `auto-completion()` will attempt to find results that match all of them. -- The `depend_on_parameter_values` parameter is a list of values for the parameters that the `auto-completion()` depends on. These values will be used to generate the auto-completion results. The type of each parameter value is the same as in the init method, which means that if a `password` parameter is specified, the type of the parameter value will be of `Password` Type. +- The `depend_on_parameter_values` parameter is a list of values for the parameters that the `auto-completion()` depends on. These values will be used to generate the auto-completion results. The type of each parameter value is the same as in the init method, which means that if a `password` parameter is specified, the type of the parameter value will be of `Password` Type. -- The `context` parameter represents the `PluginContext` in which the auto-completion is requested. This could be, for example, the context of a specific plugin, or the context of the entire system. +- The `context` parameter represents the `PluginContext` in which the auto-completion is requested. This could be, for example, the context of a specific plugin, or the context of the entire system. #### Autocompletion The method returns a list of `Autocompletion` objects, which represent the possible auto-completion results. Each `Autocompletion` object has two attributes: value and label. -- The `value` attribute represents the value to which the parameter value should be set. -- The `label` attribute is an optional label that a human user would see instead. +- The `value` attribute represents the value to which the parameter value should be set. +- The `label` attribute is an optional label that a human user would see instead. !!! Note @@ -424,10 +424,10 @@ The `system` attribute is of type SystemContext and contains general system info ExecutionContext combines context objects that are available during plugin execution. It contains four attributes: -- `system`: An instance of the SystemContext, which provides general system information. -- `user`: An optional instance of the UserContext, which provides information about the user that issued the plugin execution. -- `task`: An instance of the TaskContext, which provides metadata about the executed plugin. -- `report`: An instance of the ReportContext, which allows to update the execution report. +- `system`: An instance of the SystemContext, which provides general system information. +- `user`: An optional instance of the UserContext, which provides information about the user that issued the plugin execution. +- `task`: An instance of the TaskContext, which provides metadata about the executed plugin. +- `report`: An instance of the ReportContext, which allows to update the execution report. The ExecutionContext is used to provide context information to plugins during execution, enabling plugins to access information about the environment in which they are running, the user who initiated the execution, and the task being executed. The ReportContext attribute allows plugins to generate and update reports during execution. @@ -666,9 +666,9 @@ This JVM-based logger will prefix all plugin logs with `plugins.python. -- :material-download-circle-outline: [Installation and Usage](installation/index.md) +- :material-download-circle-outline: [Installation and Usage](installation/index.md) --- Intended for Linked Data Experts and Deployment Engineers, this page outlines how to install and use existing python plugins. -- :material-code-json: [Development](development/index.md) +- :material-code-json: [Development](development/index.md) --- Intended for Developers, this page gives an overview on the plugin concepts and how to start developing your own plugins. -- :material-cog-outline: [Setup and Configuration](setup/index.md) +- :material-cog-outline: [Setup and Configuration](setup/index.md) --- diff --git a/docs/develop/python-plugins/installation/index.md b/docs/develop/python-plugins/installation/index.md index 77066528..7b4a7389 100644 --- a/docs/develop/python-plugins/installation/index.md +++ b/docs/develop/python-plugins/installation/index.md @@ -31,7 +31,7 @@ cmem_plugin_graphql-GraphQLPlugin WorkflowPlugin GraphQL query You can get a list of all installed python packages: (1) { .annotate } -1. This list contains all installed packages in the python environment, not just your plugin packages. +1. This list contains all installed packages in the python environment, not just your plugin packages. ```shell-session title="List all installed python packages:" $ cmemc admin workspace python list @@ -95,4 +95,3 @@ Drag and drop it on the canvas, and connect it with ingoing and / or outgoing li A workaround for this is to install or upgrade plugins right after rebooting DataIntergration (and before you start a workflow that uses a python plugin). We currently advise against using NFS on this path directly. The problem is [known by the python community](https://github.com/pypa/pip/issues/6327) but there is no fix or workaround available yet. - diff --git a/docs/develop/python-plugins/setup/index.md b/docs/develop/python-plugins/setup/index.md index 4280234f..be77a32a 100644 --- a/docs/develop/python-plugins/setup/index.md +++ b/docs/develop/python-plugins/setup/index.md @@ -4,18 +4,17 @@ icon: material/cog-outline tags: - Python --- + # Setup and Configuration This section describes which backend components are needed on the Build (DataIntegration) server, in order to use python plugins. - ## Basic Configuration and Dependencies !!! info When using the official eccenca docker images, setup and basic configuration is already done. - ??? note "Build (DataIntegration) Configuration" The following Build (DataIntegration) configuration section describes how to setup and enable the Python Plugin system. @@ -46,7 +45,6 @@ This section describes which backend components are needed on the Build (DataInt } ``` - ??? note "Python Interpreter" An installation of the CPython distribution (at least version 3.3) is required. @@ -76,12 +74,12 @@ This section describes which backend components are needed on the Build (DataInt The basic setup allows for installation of packages from the [pypi.org](https://pypi.org/search/?q=%22cmem-plugin-%22) python package index, maintained by the [Python Software Foundation](https://www.python.org/psf-landing/). In order to change the remote index server, from where you can install python packages, you need to set the following environment variables in the data integration container: -- `UV_DEFAULT_INDEX` - Base URL of the default python package index. This should point to a repository which is compliant with [PEP 503 (the simple repository API)](https://peps.python.org/pep-0503/). If this variable is not set, the [official Python Package Index](https://pypi.python.org/simple) is used. - - Changing this value means, that you can install packages **only** from this repository. -- `UV_INDEX` - Extra URLs of package indexes to use in addition to the default package index. - - Example Value: `https://pypi.eccenca.com/simple https://example.org/simple` - - Multiple index URLs have to be given space-separated. - - Changing this values means you can install packages from the given repositories **in addition** to the default index. +- `UV_DEFAULT_INDEX` - Base URL of the default python package index. This should point to a repository which is compliant with [PEP 503 (the simple repository API)](https://peps.python.org/pep-0503/). If this variable is not set, the [official Python Package Index](https://pypi.python.org/simple) is used. + - Changing this value means, that you can install packages **only** from this repository. +- `UV_INDEX` - Extra URLs of package indexes to use in addition to the default package index. + - Example Value: `https://pypi.eccenca.com/simple https://example.org/simple` + - Multiple index URLs have to be given space-separated. + - Changing this values means you can install packages from the given repositories **in addition** to the default index. For individual needs, you can use additional environment variables known by `uv` (e.g. `SSL_CERT_FILE`). Please have a look at the [uv documentation](https://docs.astral.sh/uv/reference/environment/). @@ -91,9 +89,9 @@ Please have a look at the [uv documentation](https://docs.astral.sh/uv/reference In cases, where you have limited or disabled network capabilities to the internet, you can disable package retrieval and provide the packages in a local directory. To do so, you need to set the following environment variables in the data integration container: -- `UV_OFFLINE` - set the value as `true` to disable the package retrieval completely. -- `UV_FIND_LINKS` - set to a container internal directory, where the packages and its dependencies will be provided. - - Example Value: `/data/downloaded-packages` +- `UV_OFFLINE` - set the value as `true` to disable the package retrieval completely. +- `UV_FIND_LINKS` - set to a container internal directory, where the packages and its dependencies will be provided. + - Example Value: `/data/downloaded-packages` This setup will allow installation of packages and its dependencies ONLY from the given directory. @@ -102,7 +100,7 @@ To do so, use the [`pip download`](https://pip.pypa.io/en/stable/cli/pip_downloa ??? note "Example shell session showing the usage of `pip download`" - ``` + ```text $ cat requirements.txt cmem-plugin-validation cmem-plugin-graphql @@ -133,4 +131,3 @@ This is tested on Build (DataIntegration) startup. By setting environment variable `PYTHONPATH_FAILURE` (default: `true`) to other values than `true` this behavior can be skipped. However this might effect the usability of python plugins. - diff --git a/docs/explore-and-author/bke-module/visually-authoring-ontologies/index.md b/docs/explore-and-author/bke-module/visually-authoring-ontologies/index.md index cd6859b2..fcc002f0 100644 --- a/docs/explore-and-author/bke-module/visually-authoring-ontologies/index.md +++ b/docs/explore-and-author/bke-module/visually-authoring-ontologies/index.md @@ -84,9 +84,9 @@ New elements can be created from the entries listed in **Classes** on the left s ![](classes.png){ class="bordered" width="50%" } -!!! info + !!! info - If you do not see **Class** in the first entries, use the search bar to find it. + If you do not see **Class** in the first entries, use the search bar to find it. 2. Click the newly created **Untitled (Class)** in the canvas to bring up a form on the right side. diff --git a/docs/explore-and-author/charts-catalog/index.md b/docs/explore-and-author/charts-catalog/index.md index 6c4eecc1..b4bc8642 100644 --- a/docs/explore-and-author/charts-catalog/index.md +++ b/docs/explore-and-author/charts-catalog/index.md @@ -37,12 +37,12 @@ Select a query from the :eccenca-application-queries: [Query Catalog](../query-m The following activities can be performed in this component: -- **Select a query** — Select a query to visualize by clicking on a :material-plus-circle-outline: button. +- **Select a query** — Select a query to visualize by clicking on a :material-plus-circle-outline: button. The Assisted chart form (see below) supports a single query, while the Advanced chart form can use multiple queries. -- From the dropdown menu: - - **Parameters** — Some queries have parameters that need to be filled with real values. - - **Preview** — View a preview of the retrieved data. - - **View in query catalog** — Opens the query in the query catalog. +- From the dropdown menu: + - **Parameters** — Some queries have parameters that need to be filled with real values. + - **Preview** — View a preview of the retrieved data. + - **View in query catalog** — Opens the query in the query catalog. ### Chart forms @@ -74,6 +74,6 @@ To do so, add :material-plus-circle-outline: the **Chart Visualization** propert To customize the chart several placeholders can be used in you queries: -- `{{shuiResource}}`, the resource currently shown with the node shape of this property shape. -- `{{shuiMainResource}}`, refers to the main resource rendered in the start node shape of the currently displayed node shape tree (only relevant in case of sub-shape usage). -- `{{shuiGraph}}`, the currently used graph. +- `{{shuiResource}}`, the resource currently shown with the node shape of this property shape. +- `{{shuiMainResource}}`, refers to the main resource rendered in the start node shape of the currently displayed node shape tree (only relevant in case of sub-shape usage). +- `{{shuiGraph}}`, the currently used graph. diff --git a/docs/explore-and-author/companion/index.md b/docs/explore-and-author/companion/index.md index 1646104a..424d16f9 100644 --- a/docs/explore-and-author/companion/index.md +++ b/docs/explore-and-author/companion/index.md @@ -32,14 +32,14 @@ You can open the _Chat with Companion_ from any explore module via the :eccenca- In the companion sidebar, use: -- :eccenca-toggler-maximize: to increase the size of the sidebar -- :eccenca-toggler-minimize: to decrease the size of the sidebar -- :eccenca-item-add-artefact: to start a new conversation -- :octicons-triangle-down-24: the dropdown menu to select or search for a conversation -- :eccenca-item-upload: to add documents as context to the chat (supported formats include `pdf`, `txt` and `md`) -- :eccenca-item-wrench: to configure the tools that the LLM can use to answer. -- :eccenca-item-microphone: to use your microphone to dictate your questions -- :material-send-variant-outline: to submit your question +- :eccenca-toggler-maximize: to increase the size of the sidebar +- :eccenca-toggler-minimize: to decrease the size of the sidebar +- :eccenca-item-add-artefact: to start a new conversation +- :octicons-triangle-down-24: the dropdown menu to select or search for a conversation +- :eccenca-item-upload: to add documents as context to the chat (supported formats include `pdf`, `txt` and `md`) +- :eccenca-item-wrench: to configure the tools that the LLM can use to answer. +- :eccenca-item-microphone: to use your microphone to dictate your questions +- :material-send-variant-outline: to submit your question To maximize the relevance and accuracy of the generated answer, your question is automatically enriched based on the context of the current view (e.g. the query text, the graph and the resource IRI). @@ -51,7 +51,7 @@ The companion chat uses various tools to interact with Corporate Memory. These tools are used autonomously by the configured LLM model to gather the relevant context needed to generate the best answer. If your use case or question requires you to limit the available tools (e.g. if you only want to use queries from the query catalogue and not generate any), you can configure the tools the model can use via the drop-down widget. -- Query Catalog related +- Query Catalog related `query_list` : Lists all existing SPARQL queries on this corporate memory instance. @@ -63,13 +63,13 @@ If your use case or question requires you to limit the available tools (e.g. if : Executes a saved SPARQL query on this corporate memory instance. The query is identified by its IRI. -- Query related: +- Query related: `query_execute_json` : Executes a `SPARQL SELECT` query on this corporate memory instance. The query is provided as a string. -- Introspection related: +- Introspection related: `graph_tree` : Retrieves a tree of all the direct and transitive `owl:imports` for a given graph. @@ -94,7 +94,7 @@ If your use case or question requires you to limit the available tools (e.g. if : All Classes defined in this knowledge graph by installed vocabularies. While all classes are returned, they can easily be grouped by either their explicit association with an ontology in the isDefinedBy (`rdfs:isDefinedBy`) field, or by the sourceGraph field, which contains the names of the graphs from which the type statements were loaded. -- Resource related +- Resource related `resource_search` : Performs a lucene syntax full text search of labels and descriptions for 'searchTerm' and retrieve list of best matches. diff --git a/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/datatype-reference/index.md b/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/datatype-reference/index.md index c2769fbb..3a298cb8 100644 --- a/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/datatype-reference/index.md +++ b/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/datatype-reference/index.md @@ -4,7 +4,9 @@ tags: - Reference - Vocabulary --- + # Datatypes + This is a list of supported data types in shapes. @@ -14,14 +16,14 @@ This is a list of supported data types in shapes. Not all datatypes result in specific widgets. -#### anyURI +## anyURI The ·lexical space· of anyURI is finite-length character sequences which, when the algorithm defined in Section 5.4 of [XML Linking Language] is applied to them, result in strings which are legal URIs according to [RFC 2396], as amended by [RFC 2732]. Note: Spaces are, in principle, allowed in the ·lexical space· of anyURI, however, their use is highly discouraged (unless they are encoded by %20). IRI: `http://www.w3.org/2001/XMLSchema#anyURI` -#### base64Binary +### base64Binary The lexical forms of base64Binary values are limited to the 65 characters of the Base64 Alphabet defined in [RFC 2045], i.e., a-z, A-Z, 0-9, the plus sign (+), the forward slash (/) and the equal sign (=), together with the characters defined in [XML 1.0 (Second Edition)] as white space. No other characters are allowed. @@ -307,4 +309,3 @@ IRI: `http://www.w3.org/2001/XMLSchema#unsignedShort` The datatype of XML literal values. IRI: `http://www.w3.org/1999/02/22-rdf-syntax-ns#XMLLiteral` - diff --git a/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/node-shapes/index.md b/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/node-shapes/index.md index 1fc55b6f..05e53913 100644 --- a/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/node-shapes/index.md +++ b/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/node-shapes/index.md @@ -6,7 +6,9 @@ tags: - Reference - Vocabulary --- + # Node Shapes + Node Shapes are resources of type `shacl:NodeShape`. @@ -147,7 +149,7 @@ The following placeholder can be used in the query text of the SPARQL query: - `{{shuiAccount}}` - the account IRI of the active user, this includes the username (use a SUBSTR() function if you need the name only) - `{{shuiAccountName}}` - the user name/ID of the active user account - `{{shuiMainResource}}` - refers to the main resource rendered in the start node shape of the currently displayed node shape tree (only relevant in case of sub-shape usage) - + Used Path: `shui:onDeleteUpdate` diff --git a/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/property-shapes/index.md b/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/property-shapes/index.md index 5f94f7ee..c1f20681 100644 --- a/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/property-shapes/index.md +++ b/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/property-shapes/index.md @@ -4,7 +4,9 @@ tags: - Reference - Vocabulary --- + # Property Shapes + Property Shapes are resources of type `shacl:PropertyShape`. @@ -333,7 +335,7 @@ The following placeholder can be used in the query text of the SPARQL query: - `{{shuiMainResource}}` - refers to the main resource rendered in the start node shape of the currently displayed node shape tree (only relevant in case of sub-shape usage) - `{{shuiObject}}` - the object value of the statement matched by the property shape - `{{shuiProperty}}` - the IRI of the property of the statement matched by the property shape - + Used Path: `shui:onInsertUpdate` @@ -352,7 +354,7 @@ The following placeholder can be used in the query text of the SPARQL query: - `{{shuiMainResource}}` - refers to the main resource rendered in the start node shape of the currently displayed node shape tree (only relevant in case of sub-shape usage) - `{{shuiObject}}` - the object value of the statement matched by the property shape - `{{shuiProperty}}` - the IRI of the property of the statement matched by the property shape - + Used Path: `shui:onDeleteUpdate` diff --git a/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/workflow-trigger/index.md b/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/workflow-trigger/index.md index 11af5820..bb2605f5 100644 --- a/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/workflow-trigger/index.md +++ b/docs/explore-and-author/graph-exploration/building-a-customized-user-interface/workflow-trigger/index.md @@ -22,11 +22,11 @@ A workflow trigger resource references a data integration workflow by URI. To define a workflow trigger the following information is needed: -- **Label**: The trigger resource needs a label (can be given in different languages), which is used for the button presentation. -- **Description**: The trigger resource needs a description, which is used as text that is sitting left of the button for further documentation of the activity to the user. -- **Workflow**: the workflow parameter defines the workflow that shall be executed upon clicking the button. The workflow can be selected from a dropdown list. -- **Refresh View**: can be either `true `or `false`*.* If this value is set to `true`, the view that contains the workflow trigger will be reloaded upon workflow completion -- **Send Resource Reference**: can be either `true `or `false`. If this value is set to `true`, a payload that consists of the *resource IRI* that is represented in the view as well as the *graph IRI *of the graph that is currently selected*.* +- **Label**: The trigger resource needs a label (can be given in different languages), which is used for the button presentation. +- **Description**: The trigger resource needs a description, which is used as text that is sitting left of the button for further documentation of the activity to the user. +- **Workflow**: the workflow parameter defines the workflow that shall be executed upon clicking the button. The workflow can be selected from a dropdown list. +- **Refresh View**: can be either `true`or `false`*.* If this value is set to `true`, the view that contains the workflow trigger will be reloaded upon workflow completion +- **Send Resource Reference**: can be either `true`or `false`. If this value is set to `true`, a payload that consists of the *resource IRI* that is represented in the view as well as the *graph IRI *of the graph that is currently selected*.* ## Integration @@ -49,5 +49,4 @@ Workflow Payload ``` - `graphIRI` is the IRI of the graph that is currently viewed, and -- `resourceIRI `is the IRI of the resource that is viewed. - +- `resourceIRI`is the IRI of the resource that is viewed. diff --git a/docs/explore-and-author/graph-exploration/graph-insights/concepts.md b/docs/explore-and-author/graph-exploration/graph-insights/concepts.md index f82ea418..329270e8 100644 --- a/docs/explore-and-author/graph-exploration/graph-insights/concepts.md +++ b/docs/explore-and-author/graph-exploration/graph-insights/concepts.md @@ -100,4 +100,4 @@ Narrow down any group to match your current focus using [data property filters]( - **Images:** [Export](features/persistence.md#export-visuals) the canvas as SVG or PNG images. - **CSV data:** Download of the resources of a group or of the table view of a group or connection. - **Query snippets:** Copy the SPARQL that returns the resources of a group or the data in the table view of a group or connection for use (e.g. a dashboard). -- **[JSON Sharing](features/persistence.md#json-sharing):** Share explorations as JSON-serialization. \ No newline at end of file +- **[JSON Sharing](features/persistence.md#json-sharing):** Share explorations as JSON-serialization. diff --git a/docs/explore-and-author/graph-exploration/graph-insights/features/application-settings.md b/docs/explore-and-author/graph-exploration/graph-insights/features/application-settings.md index 0ed5c26b..314d1922 100644 --- a/docs/explore-and-author/graph-exploration/graph-insights/features/application-settings.md +++ b/docs/explore-and-author/graph-exploration/graph-insights/features/application-settings.md @@ -15,8 +15,8 @@ Access global configuration via the top-left dropdown menu. ### Troubleshoot and Reset -1. **Reset local data:** Clears browser-side preferences (e.g., columns visibility, width, and position). -2. **Reset graph data:** Performs a "Factory Reset" of the analysis for the current dataset. +1. **Reset local data:** Clears browser-side preferences (e.g., columns visibility, width, and position). +2. **Reset graph data:** Performs a "Factory Reset" of the analysis for the current dataset. !!! danger "Destructive Action" @@ -42,4 +42,4 @@ When no group or connection is selected, the right pane displays the **database ![Database Summary Panel](../assets/database-summary.png){ class="bordered" width="30%" } -**Sorting:** Use the `Sort by` dropdown to order lists by `Name` (alphabetical) or `Quantity` (cardinality) to identify the largest entities. \ No newline at end of file +**Sorting:** Use the `Sort by` dropdown to order lists by `Name` (alphabetical) or `Quantity` (cardinality) to identify the largest entities. diff --git a/docs/explore-and-author/graph-exploration/graph-insights/features/canvas-ui.md b/docs/explore-and-author/graph-exploration/graph-insights/features/canvas-ui.md index cdabb21a..cdda1653 100644 --- a/docs/explore-and-author/graph-exploration/graph-insights/features/canvas-ui.md +++ b/docs/explore-and-author/graph-exploration/graph-insights/features/canvas-ui.md @@ -48,6 +48,7 @@ If a group contains *more* resources than the limit, it is rendered as a single ### Instance Rendering (Count ≤ Threshold) If the count is *below* the limit, each resource is rendered as a dot within the group disk. + - The number inside each dot represents its [predecessor](objects.md#predecessors) count (number of resources in the left group linked by the connection). No number means there is only a single predecessor. - The resources with higher predecessor counts are located at the center of the group. @@ -69,4 +70,4 @@ Locate specific resources **among those currently visible** in the groups of the Use the `Temporary flags` menu that appears on the top left of the canvas to manage the persistence of the temporary flags. ---- \ No newline at end of file +--- diff --git a/docs/explore-and-author/graph-exploration/graph-insights/features/category-tree.md b/docs/explore-and-author/graph-exploration/graph-insights/features/category-tree.md index 4d4a68e7..19122bc4 100644 --- a/docs/explore-and-author/graph-exploration/graph-insights/features/category-tree.md +++ b/docs/explore-and-author/graph-exploration/graph-insights/features/category-tree.md @@ -1,6 +1,6 @@ -# All Explorations Start from the Class Tree +# All Explorations Start from the Class Tree The left pane is the **starting point of any exploration**, whether you want to **begin a fresh exploration** with a group from a class of the taxonomy or **recall a pre-existing exploration**. @@ -25,7 +25,7 @@ The tree displays the hierarchical taxonomy of the dataset: The tree is the primary tool to populate the canvas: -- **Drag and drop (Start):** Dragging a class to an empty area clears the canvas and starts a new exploration with a group of the selected class. +- **Drag and drop (Start):** Dragging a class to an empty area clears the canvas and starts a new exploration with a group of the selected class. *Alternatives: Double-click the class or use its context menu item `Start exploration with class`.* - **Drag and drop (Intersect):** Drag a class onto an **existing** group to apply an intersection filter (e.g., dragging `German Suppliers` onto `VIP Suppliers` restricts the group to resources with **both classes**). @@ -54,4 +54,4 @@ The tree is the primary tool to populate the canvas: ## Class Tree Configuration Reset -To return the class tree configuration to its initial state, click the **circular arrow icon** in the pane header. This action restores all deactivated classes and reverts caption settings to defaults. \ No newline at end of file +To return the class tree configuration to its initial state, click the **circular arrow icon** in the pane header. This action restores all deactivated classes and reverts caption settings to defaults. diff --git a/docs/explore-and-author/graph-exploration/graph-insights/features/connections.md b/docs/explore-and-author/graph-exploration/graph-insights/features/connections.md index 927f1a33..3d8e8b4b 100644 --- a/docs/explore-and-author/graph-exploration/graph-insights/features/connections.md +++ b/docs/explore-and-author/graph-exploration/graph-insights/features/connections.md @@ -29,7 +29,7 @@ When a group with visible resources is expanded, or when a connected group direc ## Left Group Restriction (Backpropagation) {#backpropagation} -By default, the exploration applies a **left-join pattern**: The left group remains static, while the right group contains the connected resources. The left group restriction switches (permanently) to an **inner-join pattern**, removing from the left group all resources having no successor in the right group. +By default, the exploration applies a **left-join pattern**: The left group remains static, while the right group contains the connected resources. The left group restriction switches (permanently) to an **inner-join pattern**, removing from the left group all resources having no successor in the right group. - **Action:** Right-click the connection to open its context menu and select `Restrict left group`. - **Outcome:** @@ -68,8 +68,8 @@ While the **[table of a group](objects-table.md)** lists resources, the **connec To open it: -1. **Context Menu:** Right-click the connection and select `Show connection table`. -2. **Quick Action:** Click the connection to select it, then click the blue `Show connection table...` button at the bottom of the canvas. +1. **Context Menu:** Right-click the connection and select `Show connection table`. +2. **Quick Action:** Click the connection to select it, then click the blue `Show connection table...` button at the bottom of the canvas. ### Table Components @@ -85,4 +85,4 @@ The table displays one row per object property assertion with predefined and var - **Structural columns:** The resource and object property columns support **search** and **sorting**, but the filtering cannot be applied (filter the groups or select another object property instead). - **Data Property columns:** Full support with filtering like the [resource table](objects-table.md#filtering) -- **Highlighting:** Hovering over a row highlights the corresponding connected resources in the exploration if the groups are in [instance rendering mode](canvas-ui.md#object-threshold-logic). \ No newline at end of file +- **Highlighting:** Hovering over a row highlights the corresponding connected resources in the exploration if the groups are in [instance rendering mode](canvas-ui.md#object-threshold-logic). diff --git a/docs/explore-and-author/graph-exploration/graph-insights/features/groups.md b/docs/explore-and-author/graph-exploration/graph-insights/features/groups.md index c5fa6ad2..317b8a30 100644 --- a/docs/explore-and-author/graph-exploration/graph-insights/features/groups.md +++ b/docs/explore-and-author/graph-exploration/graph-insights/features/groups.md @@ -15,16 +15,16 @@ Selecting a group opens the **group details** in the right pane. The header contains: - The **group caption** that is either: - - The caption of its class if no class filter was applied + - The caption of its class if no class filter was applied - A "logical expression" of class captions describing the combination of class filter applied otherwise (see [histogram filtering](#filtering)) -- A structured **natural language description** of its contents that details all steps of the traversal. +- A structured **natural language description** of its contents that details all steps of the traversal. ### Histogram Sections {#histograms} The **histogram sections** display the distribution of classes within a group. It allows analysts to scrutinize class composition (eg. spot unexpected or missing classes) and perform logical set operations (union, intersection, difference) to restrict the contents of the groups. -- **Scope:** The panel displays counts for all **classes** (including **[user-defined classes](persistence.md#custom-categories)**) present within the selected group. - *Remark: If user-defined classes are present, a second histogram is shown for better overview* +- **Scope:** The panel displays counts for all **classes** (including **[user-defined classes](persistence.md#custom-categories)**) present within the selected group. + - *Remark:* If user-defined classes are present, a second histogram is shown for better overview - **Hierarchy:** Use the `(+)` icons to expand a class (e.g., `Confections`) to reveal the distribution of its sub-classes. - **Sorting:** Use the dropdown to order bars by `Name` (Alphabetical) or `Quantity` (Cardinality). @@ -35,7 +35,7 @@ The **histogram sections** display the distribution of classes within a group. I Flagging highlights specific subsets of data for visual inspection without modifying the structure of the exploration. - **Action:** Select one or more bars in the histogram and click the **Flag** button. -- **Outcome:** Matching resources are tagged with a (temporary) orange flag and caption. +- **Outcome:** Matching resources are tagged with a (temporary) orange flag and caption. *Hint: Use the `Temporary flags` menu that appears on the top left of the canvas to manage the persistence of the temporary flags.* ![Flagged Resources in Group](../assets/histogram-flagged-resources.png){ class="bordered" width="80%" } @@ -74,14 +74,14 @@ As filters are applied, the **group caption** updates to reflect the mathematica The **expansion menu** of a group presents the options available for adding a non-empty step to the traversal. It tells you which classes are reachable via which direction of which object properties. -1. **Action:** Click any group to open the menu. -2. **Select:** The menu presents the hierarchy of connecting (directed) **object properties**, organized by target **class**. +1. **Action:** Click any group to open the menu. +2. **Select:** The menu presents the hierarchy of connecting (directed) **object properties**, organized by target **class**. - If the data model includes sub-object properties, point the parent object property to open the sub-menu with its child object property.* -3. **Action:** Click an object property with the required direction for the selected target **class** to add a new connection beam to the exploration. +3. **Action:** Click an object property with the required direction for the selected target **class** to add a new connection beam to the exploration. ![Expansion Menu](../assets/expansion-menu.png){ class="bordered" width="65%" } -4. **Alternative:** For schemas with deep object property hierarchies (or if you know exactly which object property) use the search bar to locate specific paths. +1. **Alternative:** For schemas with deep object property hierarchies (or if you know exactly which object property) use the search bar to locate specific paths. ![Filtered Expansion Menu](../assets/expansion-menu-flattened.png){ class="bordered" width="65%" } @@ -92,8 +92,8 @@ The **expansion menu** of a group presents the options available for adding a no Right-click a group or its caption to access its context menu. - **Export group resources as CSV:** Downloads a file containing all resources of the group. -- **Copy SPARQL Query:** Copies a query that replicates the traversal up to the group and returns all its resources. -- **Edit note:** Adds a text annotation that will be persisted in [saved explorations](persistence.md#saved-explorations). +- **Copy SPARQL Query:** Copies a query that replicates the traversal up to the group and returns all its resources. +- **Edit note:** Adds a text annotation that will be persisted in [saved explorations](persistence.md#saved-explorations). - **Show resource table:** Opens the tabular view (see the [dedicated page](objects-table.md)). !!! tip "Usage" diff --git a/docs/explore-and-author/graph-exploration/graph-insights/features/index.md b/docs/explore-and-author/graph-exploration/graph-insights/features/index.md index d0f65e97..75ed17eb 100644 --- a/docs/explore-and-author/graph-exploration/graph-insights/features/index.md +++ b/docs/explore-and-author/graph-exploration/graph-insights/features/index.md @@ -6,7 +6,7 @@ Documentation for the core components of the Graph Insights interface. ## Workspace -- **[Application and dataset settings](application-settings.md):** Application fine-tuning, dataset selection. +- **[Application and dataset settings](application-settings.md):** Application fine-tuning, dataset selection. - **[Exploration canvas](canvas-ui.md):** Layout controls, information density management, and exploration tree exports. - **[Class tree](category-tree.md):** Starting an exploration, class configuration (captions/visibility). @@ -20,4 +20,4 @@ Documentation for the core components of the Graph Insights interface. ## Analysis Tools - **[Histograms](groups.md#histograms):** class distribution analysis and facet-like set operations (union, intersection, difference). -- **[Persistence](persistence.md):** Saved explorations, user-defined classes, and JSON sharing. \ No newline at end of file +- **[Persistence](persistence.md):** Saved explorations, user-defined classes, and JSON sharing. diff --git a/docs/explore-and-author/graph-exploration/graph-insights/features/objects-table.md b/docs/explore-and-author/graph-exploration/graph-insights/features/objects-table.md index a45926fd..6b26cfbb 100644 --- a/docs/explore-and-author/graph-exploration/graph-insights/features/objects-table.md +++ b/docs/explore-and-author/graph-exploration/graph-insights/features/objects-table.md @@ -32,10 +32,10 @@ The resource table is the engine for refining the exploration tree. Apply filter Filtering is a two-step process to prevent unnecessary database queries. -1. **Define and preview:** Set criteria in the column headers. +1. **Define and preview:** Set criteria in the column headers. - **Outcome:** The table updates immediately to preview the result, and the column header displays an **hourglass icon (⧖)** (symbolizes filtering). - Repeat with further columns as necessary. -2. **Apply:** Click `Apply filters` (top-left). +2. **Apply:** Click `Apply filters` (top-left). - **Outcome:** The group on the canvas updates, and a **half full disk icon (◐)** appears on the group (indicating filtered content). ![Filter Status Icons](../assets/filtered-group-and-column.png){ class="bordered" width="85%" } diff --git a/docs/explore-and-author/graph-exploration/graph-insights/features/objects.md b/docs/explore-and-author/graph-exploration/graph-insights/features/objects.md index 1a233ad5..5e3c46ff 100644 --- a/docs/explore-and-author/graph-exploration/graph-insights/features/objects.md +++ b/docs/explore-and-author/graph-exploration/graph-insights/features/objects.md @@ -2,7 +2,7 @@ # Resources -Most investigations require **isolating individual resources** at some point for closer inspection. This page covers the **resource details pane**, the **highlighting and flagging tools**, and **single resource traversals**. +Most investigations require **isolating individual resources** at some point for closer inspection. This page covers the **resource details pane**, the **highlighting and flagging tools**, and **single resource traversals**. --- @@ -14,11 +14,12 @@ Select a visible resource in a group or its row in the group table to open the ` ### Components -1. **Header:** Displays the **caption** and **description** , as well as the IRI of the resource. -2. **Classes:** List of all classes assigned to the resource. -3. **Data Properties:** List of all data properties organized in sections by their "usage domain" classes (see below). +1. **Header:** Displays the **caption** and **description** , as well as the IRI of the resource. +2. **Classes:** List of all classes assigned to the resource. +3. **Data Properties:** List of all data properties organized in sections by their "usage domain" classes (see below). ### Controls + - **Hide empty fields:** Toggles visibility of data properties that have no value for this specific resource. - **Copy Data:** Right-click any row to copy the **data property** or **data property assertion**. @@ -48,7 +49,7 @@ The standard flags are persistent markers used to track the appearance of specif !!! info "About Persistence" - A flag will appear anytime a new instance of the flagged resource is visible in a subsequent exploration step (even in new explorations). - - Flags and notes can be removed using the context-menu of the resource, the group, or the exploration (canvas). + - Flags and notes can be removed using the context-menu of the resource, the group, or the exploration (canvas). ### Temporary Flags {#temporary-flags} @@ -83,7 +84,7 @@ Visualize upstream and downstream connections of a resource in the exploration t **Flagging:** -- **Action:** Right-click and select `Flag all connected resources...` +- **Action:** Right-click and select `Flag all connected resources...` - **Outcome:** [Temporary flags](#temporary-flags) are added to all connected resources visible. ![Branch Highlighting](../assets/flag-predecessors-successors.png){ class="bordered" width="95%" } @@ -97,8 +98,8 @@ Visualize upstream and downstream connections of a resource in the exploration t ## Connectivity Visualization (Predecessors) {#predecessors} -In the **groups in instance rendering mode** (count <= threshold), Graph Insights visually encodes the local "centrality" of resources based on their connectivity to the previous group in the exploration tree (if any). - +In the **groups in instance rendering mode** (count <= threshold), Graph Insights visually encodes the local "centrality" of resources based on their connectivity to the previous group in the exploration tree (if any). + - **Definition:** The **number of predecessors** of a resource is the number of distinct resources from the *previous* group in the current branch of the exploration tree connected to it. - **Visualization:** - The number of predecessors is displayed in the dot representing the resources, no number meaning there is only a single predecessor. @@ -140,4 +141,4 @@ Start a new exploration tree with a specific resource found in the current explo - **Action:** Right-click the resource to open its context menu and select `Start exploration with resource`. - **Outcome:** The canvas clears and places the single resource as starting point of the exploration. -![Start Exploration with Object](../assets/start-exploration-with-object.png){ class="bordered" width="70%" } \ No newline at end of file +![Start Exploration with Object](../assets/start-exploration-with-object.png){ class="bordered" width="70%" } diff --git a/docs/explore-and-author/graph-exploration/graph-insights/features/persistence.md b/docs/explore-and-author/graph-exploration/graph-insights/features/persistence.md index 7c438e33..512ff5c8 100644 --- a/docs/explore-and-author/graph-exploration/graph-insights/features/persistence.md +++ b/docs/explore-and-author/graph-exploration/graph-insights/features/persistence.md @@ -25,7 +25,7 @@ To save the current exploration, either right-click the canvas background to ope ### Saved Exploration Folders The saved explorations folders behave like a usual file system: - + - Use the **context menu of the folders** in the saved explorations section of the left pane **to create new subfolders**. - New saved explorations are placed **in the folder highlighted at the time of the operation.** - **Names** must only be **unique within their folder**. @@ -39,7 +39,7 @@ If you load and modify a loaded exploration (e.g., add a filter), Graph Insights ### Annotations (Group Notes) -Add context to the analysis by attaching text notes to important groups. +Add context to the analysis by attaching text notes to important groups. - **Add note:** Right-click a group to open its context menu and select `Add note`. - **Toggle visibility:** Use the group context menu to toggle the visibility of a single note, or use the context menu of the canvas (or the `Exploration` top menu) and select `Hide/Show all group notes`. @@ -52,7 +52,7 @@ Add context to the analysis by attaching text notes to important groups. --- - + ## User-Defined Classes {#custom-categories} **User-defined Graph Insights classes** are customizable tools designed to analyze data with specific goals (e.g. identifying high-risk suppliers based at a given location and having a specific contract type). @@ -63,7 +63,7 @@ They act as **"virtual classes"**, refining the model within the exploration lay A user-defined Graph Insights class can be defined for any group in any exploration (e.g., "High-volume suppliers located in Germany"). - **Action:** Right-click the group or its caption to open its context menu and select `Add Graph Insights class for this group`. -- **Outcome:** +- **Outcome:** - The group is relabeled with the name of the new class. - The new class is **added to the class tree** and the Graph Insights classes list in the right pane, and is handled like a standard class. @@ -89,9 +89,9 @@ A user-defined Graph Insights class can be defined for any group in any explorat Share an exploration serialized as a JSON string. -1. **Export:** Right-click the canvas to open the exploration context menu or use the `Exploration` top menu, then select `Copy exploration as JSON`. -2. **Share:** Send the text blob to another user. -3. **Import:** The recipient selects `Start exploration from JSON` from the canvas context menu or the `Exploration` top menu, and pastes the string. +1. **Export:** Right-click the canvas to open the exploration context menu or use the `Exploration` top menu, then select `Copy exploration as JSON`. +2. **Share:** Send the text blob to another user. +3. **Import:** The recipient selects `Start exploration from JSON` from the canvas context menu or the `Exploration` top menu, and pastes the string. !!! warning "Limitations" @@ -107,4 +107,4 @@ Export the current canvas state for reporting. - **Access:** Right-click the canvas background or use the `Exploration` top menu. - **Formats:** - **Raster (PNG):** For slides and quick sharing. - - **Vector (SVG):** For high-resolution print or editing. \ No newline at end of file + - **Vector (SVG):** For high-resolution print or editing. diff --git a/docs/explore-and-author/graph-exploration/graph-insights/index.md b/docs/explore-and-author/graph-exploration/graph-insights/index.md index b3d53490..889738f0 100644 --- a/docs/explore-and-author/graph-exploration/graph-insights/index.md +++ b/docs/explore-and-author/graph-exploration/graph-insights/index.md @@ -50,13 +50,13 @@ The **user guide** presents the visual analysis workflow of Graph Insights. It d ### Where to Start? -1. **[Core Concepts](concepts.md):** *(Recommended)* Before diving in, understand the **aggregated exploration tree** paradigm of Graph Insights that prevents the visual clutter of traditional force-directed graph renderings ("hairballs"). +1. **[Core Concepts](concepts.md):** *(Recommended)* Before diving in, understand the **aggregated exploration tree** paradigm of Graph Insights that prevents the visual clutter of traditional force-directed graph renderings ("hairballs"). -2. **[Tutorial: Supply Chain Analysis](tutorial.md):** A step-by-step walkthrough using the Northwind dataset. Learn how to filter for **VIP customers**, analyze **product vulnerabilities**, and use **backpropagation** to propagate filtering. +2. **[Tutorial: Supply Chain Analysis](tutorial.md):** A step-by-step walkthrough using the Northwind dataset. Learn how to filter for **VIP customers**, analyze **product vulnerabilities**, and use **backpropagation** to propagate filtering. -3. **[Feature Reference](features/index.md):** Detailed documentation for every component of the interface. +3. **[Feature Reference](features/index.md):** Detailed documentation for every component of the interface. - **Workspace:** [Canvas](features/canvas-ui.md), [Search](features/category-tree.md), and [Settings](features/application-settings.md#settings). - **Interaction:** [Groups](features/groups.md), [Connections](features/connections.md), and [Details](features/objects.md#object-details). - **Analysis:** [Histograms](features/groups.md#histograms) and [Persistence](features/persistence.md). -4. **[Platform Integration](navigating-to-cmem.md):** Learn how to seamlessly jump from a graph node to other Corporate Memory views. +4. **[Platform Integration](navigating-to-cmem.md):** Learn how to seamlessly jump from a graph node to other Corporate Memory views. diff --git a/docs/explore-and-author/graph-exploration/graph-insights/navigating-to-cmem.md b/docs/explore-and-author/graph-exploration/graph-insights/navigating-to-cmem.md index d090e94a..7a9e019a 100644 --- a/docs/explore-and-author/graph-exploration/graph-insights/navigating-to-cmem.md +++ b/docs/explore-and-author/graph-exploration/graph-insights/navigating-to-cmem.md @@ -28,14 +28,14 @@ If you isolate a specific resource of interest during your exploration, you can There are two ways to trigger this navigation: -**From the Canvas** +### From the Canvas - **Action:** Right-click a specific resource dot within a group to open its context menu. - **Select:** `View in CMEM`. ![View Resource in Corporate Memory from Canvas](graph-insights-view-in-cmem-resources.png){ class="bordered" width="75%" } -**From the Details Pane** +### From the Details Pane - **Action:** Select a resource on the canvas or in the resource table to load its details in the right pane. - **Select:** Click the `View in CMEM` button located at the top of the Details pane. diff --git a/docs/explore-and-author/graph-exploration/graph-insights/tutorial.md b/docs/explore-and-author/graph-exploration/graph-insights/tutorial.md index 55988675..70655a0b 100644 --- a/docs/explore-and-author/graph-exploration/graph-insights/tutorial.md +++ b/docs/explore-and-author/graph-exploration/graph-insights/tutorial.md @@ -43,9 +43,9 @@ We will traverse the ordering chain, identify high-volume US partners, and isola Every exploration requires a starting point. We begin with the central entity: The **product**. -1. Locate the `Product` **class** in the [class tree](features/category-tree.md) in the left pane. -2. Drag it to the center of the canvas. -3. **Outcome:** A group appears, representing the entire inventory (e.g., 77 Products). +1. Locate the `Product` **class** in the [class tree](features/category-tree.md) in the left pane. +2. Drag it to the center of the canvas. +3. **Outcome:** A group appears, representing the entire inventory (e.g., 77 Products). ![Start with Product Category](assets/tutorial-start-with-product.png){ class="bordered" width="80%" } @@ -58,6 +58,7 @@ Next, we expand the exploration tree to connect Suppliers and Customers. ### Preparation To simplify the expansion menus, hide generic classes that are not relevant to this analysis. + - Right-click the `Agent` class in the class tree. - Select `Deactivate class`. @@ -65,11 +66,11 @@ To simplify the expansion menus, hide generic classes that are not relevant to t We extend the exploration tree from the central `Product` group. -1. **Add suppliers:** +1. **Add suppliers:** - Select the `Product` group. - Expand to the `Supplier` class via `→ hasSupplier`. -2. **Add customers:** +2. **Add customers:** - Select the `Product` group again. - Expand to the `Order` class via `← includesProduct`. - From the new `Order` group, expand to the `Customer` class via `→ hasCustomer`. @@ -83,23 +84,24 @@ We extend the exploration tree from the central `Product` group. The exploration tree currently visualizes all connections. We must now isolate the high-value targets (VIPs) and the specific risk region (UK). ### 3.1. Identify VIP US Customers + We define "VIP" as customers with the highest volume of incoming orders. -1. Select the `Customer` group and open the [resource table](features/objects-table.md). -2. **Filter by country:** Open the menu of the `country` column from its header (using the menu dropdown or a right-click) and enter `USA` in the text field of the filter submenu. -3. **Identify VIPs:** Click the **Predecessors** column header to **Sort Descending**. This ranks customers by their incoming connection count (number of Orders). +1. Select the `Customer` group and open the [resource table](features/objects-table.md). +2. **Filter by country:** Open the menu of the `country` column from its header (using the menu dropdown or a right-click) and enter `USA` in the text field of the filter submenu. +3. **Identify VIPs:** Click the **Predecessors** column header to **Sort Descending**. This ranks customers by their incoming connection count (number of Orders). ![USA Customer Filter](assets/tutorial-usa-customer-filter.png){ class="bordered" width="80%" } -4. **Select and restrict:** Check the boxes for the top 5 customers and click `Restrict to selection`. +1. **Select and restrict:** Check the boxes for the top 5 customers and click `Restrict to selection`. ![Top USA Customer Filter](assets/tutorial-top-usa-customer-filter.png){ class="bordered" width="85%" } ### 3.2. Isolate UK Suppliers -1. Select the `Supplier` group to open the table. -2. Filter `Country` to `UK`. -3. Click `Apply filters`. +1. Select the `Supplier` group to open the table. +2. Filter `Country` to `UK`. +3. Click `Apply filters`. ![UK Supplier Filter](assets/tutorial-uk-suppliers-filter.png){ class="bordered" width="85%" } @@ -152,18 +154,18 @@ We have isolated the high risk product group, that we can now analyze: Finally, we convert this visual insight into an actionable list for the procurement team. -1. **Drill down:** +1. **Drill down:** - In the histogram, right-click the `Confections` bar. - Select `Restrict group to class`. This discards the other product types, leaving only the high-risk confections, with the side effect of reducing the UK suppliers to a single resource. -2. **Visualize traversals including a specific resource:** +2. **Visualize traversals including a specific resource:** - Open the resource table of the `Products` group. - Right-click a specific high-value item (e.g., "Teatime Chocolate Biscuits") to open its context menu. - Select `Flag all connected resources on the branch`. ![Supply Chain: Flag Branch](assets/tutorial-supply-chain-flag-branch-menu.png){ class="bordered" width="85%" } - - Graph Insights highlights all resources on a specific high-risk traversal: The specific UK Supplier → The specific Biscuit → The specific VIP US Customers buying it. +- Graph Insights highlights all resources on a specific high-risk traversal: The specific UK Supplier → The specific Biscuit → The specific VIP US Customers buying it. ![Supply Chain: Flagged Branch](assets/tutorial-supply-chain-flagged-branch.png){ class="bordered" width="85%" } diff --git a/docs/explore-and-author/graph-exploration/index.md b/docs/explore-and-author/graph-exploration/index.md index 2911809b..6279699e 100644 --- a/docs/explore-and-author/graph-exploration/index.md +++ b/docs/explore-and-author/graph-exploration/index.md @@ -15,13 +15,14 @@ To open the Explore module, click **:eccenca-application-explore: Knowledge G The user interface of the Explore module shows the following main areas:
-- the header area, showing: - - selected elements, - - possible actions (e.g. **:eccenca-item-add-artefact: create** or **:eccenca-item-remove: remove resource**), - - a **:eccenca-module-search: Go to resource** (2) input field, - - and a **:eccenca-application-useraccount: user menu** -- the navigation area, showing the [Graphs](#graphs) and the [Navigation](#navigation) structures, (1) -- the main area, providing multiple views, depending on which resource has been selected. + +- the header area, showing: + - selected elements, + - possible actions (e.g. **:eccenca-item-add-artefact: create** or **:eccenca-item-remove: remove resource**), + - a **:eccenca-module-search: Go to resource** (2) input field, + - and a **:eccenca-application-useraccount: user menu** +- the navigation area, showing the [Graphs](#graphs) and the [Navigation](#navigation) structures, (1) +- the main area, providing multiple views, depending on which resource has been selected.
1. If necessary, you can toggle the navigation area by using the @@ -45,10 +46,10 @@ In the main area, the Metadata view of the selected graph appears, showing sev The Graphs are categorized into groups as follows: -- User: All graphs which represent user data (created manually or by build processes). -- Vocabularies: All graphs containing vocabularies. -- System: All graphs containing configuration data. -- All +- User: All graphs which represent user data (created manually or by build processes). +- Vocabularies: All graphs containing vocabularies. +- System: All graphs containing configuration data. +- All You can search for a specific graph with **:eccenca-module-search: Search**. @@ -57,11 +58,12 @@ You can search for a specific graph with **:eccenca-module-search: Search**. To add a new graph to the Graphs list:
-- Click **:eccenca-item-add-artefact: Add new graph**. A dialog appears. -- Select a graph type. (1) -- Provide a name and enter the graph URI (e.g. `https://ns.eccenca.com`). -- Click **Next** and provide metadata (different types, require different metadata to enter). -- Click **Save** to create the new graph. + +- Click **:eccenca-item-add-artefact: Add new graph**. A dialog appears. +- Select a graph type. (1) +- Provide a name and enter the graph URI (e.g. `https://ns.eccenca.com`). +- Click **Next** and provide metadata (different types, require different metadata to enter). +- Click **Save** to create the new graph.
1. More concrete, you select a shape here. @@ -71,9 +73,9 @@ To add a new graph to the Graphs list: To download a graph from the Graphs list: -- In the **Graphs** list, click **:eccenca-item-download: Download graph** on the graph you want to download. -- A message box appears, stating that downloading can take a long time. -- Click **Download**. +- In the **Graphs** list, click **:eccenca-item-download: Download graph** on the graph you want to download. +- A message box appears, stating that downloading can take a long time. +- Click **Download**. ### :eccenca-item-edit: Managing a graph @@ -82,13 +84,14 @@ Use this function to add or replace data in the a graph. To update or replace data of a graph:
-- In the **Graphs** box, select **:eccenca-item-download: Manage graph** on the graph you want to update or replace. -- A dialog box appears. -- Click **Choose file** to upload a file containing the new or updated data. (1) -- Choose one of the following options: - - **Update**: add uploaded data to Graph. - - **Replace**: clear Graph and add uploaded data. -- Click **Update** to start the upload process. + +- In the **Graphs** box, select **:eccenca-item-download: Manage graph** on the graph you want to update or replace. +- A dialog box appears. +- Click **Choose file** to upload a file containing the new or updated data. (1) +- Choose one of the following options: + - **Update**: add uploaded data to Graph. + - **Replace**: clear Graph and add uploaded data. +- Click **Update** to start the upload process.
1. You can upload one of the following file formats: Turtle, N-Triples, RDF/XML, or JSON-LD. @@ -193,7 +196,7 @@ This tab shows all resources that link back to the selected resource. This tab shows the turtle RDF representation of the raw data representing the resource. You can use this tab to edit the selected resource: -- Enter your changes in turtle. -- Click **UPDATE** to save your changes. +- Enter your changes in turtle. +- Click **UPDATE** to save your changes. Deleting the entire turtle representation deletes the resource. diff --git a/docs/explore-and-author/graph-exploration/statement-annotations/index.md b/docs/explore-and-author/graph-exploration/statement-annotations/index.md index 0fb958ef..cfb39d89 100644 --- a/docs/explore-and-author/graph-exploration/statement-annotations/index.md +++ b/docs/explore-and-author/graph-exploration/statement-annotations/index.md @@ -10,9 +10,9 @@ tags: Statement Annotations provide a way to express knowledge about statements. Typical use cases for Statement Annotations include: -- the temporal validity of information, -- the origin of information, or -- just a way to annotate a specific statement with a human readable comment. +- the temporal validity of information, +- the origin of information, or +- just a way to annotate a specific statement with a human readable comment. ## Usage @@ -22,9 +22,9 @@ If enabled on a specific type of statement or type of resource, you see a Statem This bubble has different status: -- A **empty text bubble** indicates, that there is no annotation on the statement, but the annotation feature is enabled for this statement. -- A **filled text bubble** indicates, that there is at least one annotation on the statement. -- **No bubble** indicates, that the annotation feature is NOT enabled on this type of statement. +- A **empty text bubble** indicates, that there is no annotation on the statement, but the annotation feature is enabled for this statement. +- A **filled text bubble** indicates, that there is at least one annotation on the statement. +- **No bubble** indicates, that the annotation feature is NOT enabled on this type of statement. Clicking on one of the text bubbles opens the Statement Annotation dialog for this specific statement: @@ -102,4 +102,3 @@ WHERE { } } ``` - diff --git a/docs/explore-and-author/index.md b/docs/explore-and-author/index.md index b7f9d91b..47989f4c 100644 --- a/docs/explore-and-author/index.md +++ b/docs/explore-and-author/index.md @@ -18,29 +18,28 @@ You will also learn how we make use of [SHACL Shapes](https://www.w3.org/TR/sha
-- :eccenca-application-explore: [Graph Exploration](graph-exploration/index.md) +- :eccenca-application-explore: [Graph Exploration](graph-exploration/index.md) --- This module provides a generic and extensible RDF data browser and editor. -- :eccenca-application-vocabularies: [Vocabulary Catalog](vocabulary-catalog/index.md) +- :eccenca-application-vocabularies: [Vocabulary Catalog](vocabulary-catalog/index.md) --- This module allows for managing vocabularies in Corporate Memory that are accessible for the user. -- :eccenca-module-thesauri: [Thesauri Management](thesauri-management/index.md) +- :eccenca-module-thesauri: [Thesauri Management](thesauri-management/index.md) --- The Thesaurus module provides a user interface to create, browse and edit thesaurus resources and general taxonomical data modeled in SKOS. -- :eccenca-application-queries: [Query Module](query-module/index.md) +- :eccenca-application-queries: [Query Module](query-module/index.md) --- The Query module provides a user interface to store, describe, search and edit SPARQL queries.
- diff --git a/docs/explore-and-author/link-rules/index.md b/docs/explore-and-author/link-rules/index.md index 77b340e5..9c5f53f6 100644 --- a/docs/explore-and-author/link-rules/index.md +++ b/docs/explore-and-author/link-rules/index.md @@ -37,12 +37,12 @@ Click the :eccenca-toggler-showmore: toggler icon (④) next to a rule to expand In the Details tab (⑦), you’ll find: -- General Information (operators, paths) -- Result Statistics -- Rule Status -- Publication Information -- Activities -- The Custom tab contains any additional user-defined properties. +- General Information (operators, paths) +- Result Statistics +- Rule Status +- Publication Information +- Activities +- The Custom tab contains any additional user-defined properties. In the rule execution panel (⑥), you can **:eccenca-item-start: Execute** the rule, **:eccenca-item-remove: Delete** the rule, **:fontawesome-regular-clone: Clone** the rule, or **:eccenca-item-edit: Modify** its configuration. @@ -107,17 +107,17 @@ This access conditions is used by the Link Rules module to grant write access to In order to enable users to use the Link Rules Module, the following must be allowed (this can be granted in this rule or be defined separately): -- **Allow reading graph** - - Respective data graphs -- **Allow writing graph** - - *CMEM DI Project Config* graph of the "Link Rules" project - - `{IRI of the Link Rules project}-meta` (e.g., `http://di.eccenca.com/project/linkrules-meta` according to the project identifier in this example) -- **Allowed action** - - *Build - Workspace* - - *Explore - Knowledge Graph Exploration* - - *Explore - Link Rules Module* -- **Graph pattern for granting write access** - - Pattern according to the *Output Graph Template*, e.g. `http://eccenca.com/user_rules/result_*` +- **Allow reading graph** + - Respective data graphs +- **Allow writing graph** + - *CMEM DI Project Config* graph of the "Link Rules" project + - `{IRI of the Link Rules project}-meta` (e.g., `http://di.eccenca.com/project/linkrules-meta` according to the project identifier in this example) +- **Allowed action** + - *Build - Workspace* + - *Explore - Knowledge Graph Exploration* + - *Explore - Link Rules Module* +- **Graph pattern for granting write access** + - Pattern according to the *Output Graph Template*, e.g. `http://eccenca.com/user_rules/result_*` ### Workspace configuration @@ -125,18 +125,18 @@ Enable the *Link Rules* module. In the following let us walk through the configuration options. -- **Order**: The position of the Link Rules module in the menu. -- **Template Catalog IRI**: The IRI of the template catalog we just created. As the name implies, the Template Catalog holds all templates for usage in the Link Rules module. Templates allow preselecting patterns for easier rule creation. (`https://ns.eccenca.com/data/linkruletemplates/` in this example) -- **Project ID**: The *Project Identifier* of the Link Rules Build project we just created. Link Rules will be managing this project and add the rules to it. To get the id of a project, open the project in the build component, select the project and choose 'Show identifier' in the context menu, in the top right. The project needs to have a workflow, which is used to execute the rules and has to be configured in this section. Furthermore, the projects needs one or more Datasets, which can be selected in the templates. (`linkrules` in this example) -- **Access Condition IRI**: The IRI of the Access Condition we just created. Explore the `CMEM Access Conditions` graph to find the `Access Condition` rule and its respective IRI. -- **Published Rules Workflow ID**: The *Item Identifier* of the workflow in the Link Rules Build project we just created. -- **Output Graph Template**: A jinja template for the graph IRIs which will hold the results of a linking rule. It needs to include the placeholder `{name}`, for example `http://eccenca.com/user_rules/result_{name}`. -- **Publication Graph Template**: A jinja template, that generates the IRI of the graph, into which the output graph is imported via an `owl:imports`. This can be a simple graph IRI, for example `http://eccenca.com/user_rules/results_published`. -- **Result Download Query** (optional): A query that will be used to download the results of a linking rule. By default all resource IRIs will be listed as `resource1` and `resource2`. Whenever you want to provide other data via the *Download results* function, such as a specific identifier or a label instead of the IRI, this query can be adapted here. -- **Annotation Class IRI** (optional): Rule results can be enriched with annotations. Instances of this class can be selected in the link rules interface. Instances have both a connection to the rule and the result. The connection is defined by the *Annotation Link Property IRI*. The values of the annotation are defined by the *Annotation Value Property IRI*. This relation is materialized as `rdf:value` in the output graph upon execution. -- **Annotation Value Property IRI** (optional): Values of sub-properties of this property are actually connected as `rdf:values` to the generated links of a rule. -- **Annotation Link Property IRI** (optional): The connection between a rule and an annotation resource (i.e. instance of *Annotation Class IRI*) is defined by this property. -- **Annotation Graph IRI** (optional): The IRI of the graph, from which instances of the *Annotation Class IRI* are selected. +- **Order**: The position of the Link Rules module in the menu. +- **Template Catalog IRI**: The IRI of the template catalog we just created. As the name implies, the Template Catalog holds all templates for usage in the Link Rules module. Templates allow preselecting patterns for easier rule creation. (`https://ns.eccenca.com/data/linkruletemplates/` in this example) +- **Project ID**: The *Project Identifier* of the Link Rules Build project we just created. Link Rules will be managing this project and add the rules to it. To get the id of a project, open the project in the build component, select the project and choose 'Show identifier' in the context menu, in the top right. The project needs to have a workflow, which is used to execute the rules and has to be configured in this section. Furthermore, the projects needs one or more Datasets, which can be selected in the templates. (`linkrules` in this example) +- **Access Condition IRI**: The IRI of the Access Condition we just created. Explore the `CMEM Access Conditions` graph to find the `Access Condition` rule and its respective IRI. +- **Published Rules Workflow ID**: The *Item Identifier* of the workflow in the Link Rules Build project we just created. +- **Output Graph Template**: A jinja template for the graph IRIs which will hold the results of a linking rule. It needs to include the placeholder `{name}`, for example `http://eccenca.com/user_rules/result_{name}`. +- **Publication Graph Template**: A jinja template, that generates the IRI of the graph, into which the output graph is imported via an `owl:imports`. This can be a simple graph IRI, for example `http://eccenca.com/user_rules/results_published`. +- **Result Download Query** (optional): A query that will be used to download the results of a linking rule. By default all resource IRIs will be listed as `resource1` and `resource2`. Whenever you want to provide other data via the *Download results* function, such as a specific identifier or a label instead of the IRI, this query can be adapted here. +- **Annotation Class IRI** (optional): Rule results can be enriched with annotations. Instances of this class can be selected in the link rules interface. Instances have both a connection to the rule and the result. The connection is defined by the *Annotation Link Property IRI*. The values of the annotation are defined by the *Annotation Value Property IRI*. This relation is materialized as `rdf:value` in the output graph upon execution. +- **Annotation Value Property IRI** (optional): Values of sub-properties of this property are actually connected as `rdf:values` to the generated links of a rule. +- **Annotation Link Property IRI** (optional): The connection between a rule and an annotation resource (i.e. instance of *Annotation Class IRI*) is defined by this property. +- **Annotation Graph IRI** (optional): The IRI of the graph, from which instances of the *Annotation Class IRI* are selected. When the configuration is saved the *Link Rules module* should appear in the *Build* section of the menu. @@ -146,15 +146,15 @@ Templates can be managed at *Link Rules module* -> *Manage Templates*. Create a new Link Rule Template. Such a template describes an abstract link rule with pre-defined settings. -- **Label**: A mandatory label for the template. -- **Target Property**: The (default) connecting property for the derived link rules. The property can be individually set for each link rule later on. -- **Input** group: The datasets which hold the subjects (source) and objects (target) to link. -- **Source/Target Dataset**: The dataset holding the resources to link. -- **Source/Target Resource Pattern**: A filter description template for the resources to link. This is a JSON object as described in [Graph Resource Pattern](#graph-resource-pattern). When creating a Link Rule based on this template these resource patterns can be altered. So this template may remain simple, but should provide sufficient help for the end user to define a proper resource selection. -- **Output** group: The dataset which will hold the Link Rule results. -- **Output Graph**: The graph IRI where to write the Link Rule results. Available placeholders: `{name}` for the Link Rules name. +- **Label**: A mandatory label for the template. +- **Target Property**: The (default) connecting property for the derived link rules. The property can be individually set for each link rule later on. +- **Input** group: The datasets which hold the subjects (source) and objects (target) to link. +- **Source/Target Dataset**: The dataset holding the resources to link. +- **Source/Target Resource Pattern**: A filter description template for the resources to link. This is a JSON object as described in [Graph Resource Pattern](#graph-resource-pattern). When creating a Link Rule based on this template these resource patterns can be altered. So this template may remain simple, but should provide sufficient help for the end user to define a proper resource selection. +- **Output** group: The dataset which will hold the Link Rule results. +- **Output Graph**: The graph IRI where to write the Link Rule results. Available placeholders: `{name}` for the Link Rules name. -##### Graph Resource Pattern +#### Graph Resource Pattern This section specifies the JSON object used to provide a search filter configuration for source and target resources used for the Link Rule. @@ -193,15 +193,15 @@ In the `paths` section triple patterns are defined using `subjectVarName`, `pred The `pathFilters` section defines restrictions on the variables defined above. The following path filter properties can be used: -- `isNoneOfLiteral` -- `isNoneOfResource` -- `varIsAnyOneOfLiteral` -- `varIsAnyOneOfResource` -- `literalFilters` -- `GreaterThan` -- `LessThan` -- `GreaterEqualsThan` -- `LessEqualThan` -- `NotEquals` -- `Contains` -- `Regex` +- `isNoneOfLiteral` +- `isNoneOfResource` +- `varIsAnyOneOfLiteral` +- `varIsAnyOneOfResource` +- `literalFilters` +- `GreaterThan` +- `LessThan` +- `GreaterEqualsThan` +- `LessEqualThan` +- `NotEquals` +- `Contains` +- `Regex` diff --git a/docs/explore-and-author/thesauri-management/index.md b/docs/explore-and-author/thesauri-management/index.md index c20b7fb0..94128e20 100644 --- a/docs/explore-and-author/thesauri-management/index.md +++ b/docs/explore-and-author/thesauri-management/index.md @@ -149,8 +149,8 @@ You can add, for example, a second broader concept for an existing concept or a To add relations, select the concept in the navigation tree. In the detail view, click **:eccenca-item-edit: Edit** to open the edit mode. -- To add an associative relation to another concept, enter the concept name in the field **Related concept**. -- To add a further broader relation, enter the name of the broader concept in the field **Broader concepts**. +- To add an associative relation to another concept, enter the concept name in the field **Related concept**. +- To add a further broader relation, enter the name of the broader concept in the field **Broader concepts**. You can only choose from existing concepts. Click **SAVE** to confirm your changes. diff --git a/docs/explore-and-author/vocabulary-catalog/index.md b/docs/explore-and-author/vocabulary-catalog/index.md index 93910118..66ba14d6 100644 --- a/docs/explore-and-author/vocabulary-catalog/index.md +++ b/docs/explore-and-author/vocabulary-catalog/index.md @@ -30,8 +30,8 @@ A vocabulary which is known and available but not installed, looks like this: Example of extended information of uninstalled Vocabulary Catalog -- Use **Install** or the switch in the column `Installed` to install the Catalog. -- Use **View** to access the Vocabulary. +- Use **Install** or the switch in the column `Installed` to install the Catalog. +- Use **View** to access the Vocabulary. A vocabulary which is installed looks like this @@ -39,6 +39,6 @@ A vocabulary which is installed looks like this Example of extended information of installed Vocabulary Catalog -- Use **Uninstall** to remove an installed vocabulary or **Install** to install a vocabulary. -- Use **View** to access the Vocabulary. -- Use **Upload** to install or overwrite the vocabulary from a file. +- Use **Uninstall** to remove an installed vocabulary or **Install** to install a vocabulary. +- Use **View** to access the Vocabulary. +- Use **Upload** to install or overwrite the vocabulary from a file. diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md index 813a7249..6c6de8e9 100644 --- a/docs/getting-started/index.md +++ b/docs/getting-started/index.md @@ -24,21 +24,21 @@ eccenca Corporate Memory is a semantic data management software that accelerates The main features of Corporate Memory include: -- Flexible metadata and schema layer based on knowledge graphs -- Data virtualization and analytics -- Data integration and indexing -- Dataset and vocabulary management -- Thesaurus and taxonomy management -- Big data scalability -- Access control +- Flexible metadata and schema layer based on knowledge graphs +- Data virtualization and analytics +- Data integration and indexing +- Dataset and vocabulary management +- Thesaurus and taxonomy management +- Big data scalability +- Access control ### Minimal requirements For the best user experience, we recommend to use the newest version of Google Chrome or Mozilla Firefox. Corporate Memory is tested with the following browsers: -- Google Chrome 83 or later -- Mozilla Firefox 78 or later -- Microsoft Edge 83 (on Windows) or later +- Google Chrome 83 or later +- Mozilla Firefox 78 or later +- Microsoft Edge 83 (on Windows) or later ## Login and Logout @@ -71,16 +71,16 @@ To access a module, click the module name. The active module is highlighted. By default, Corporate Memory provides the following modules: -- EXPLORE - for Knowledge Graph browsing and exploration, specifically - - [Knowledge Graphs](../explore-and-author/index.md) - a generic and extensible RDF data browser and editor - - [Vocabularies](../explore-and-author/vocabulary-catalog/index.md) - for vocabulary management - - [Thesauri](../explore-and-author/thesauri-management/index.md) - for managing thesauri and taxonomies based on SKOS - - [Queries](../explore-and-author/query-module/index.md) - a SPARQL query interface -- [BUILD](../build/index.md) - for creating and integrating Knowledge Graphs, with specific links to - - Projects - the BUILD Projects level - - Datasets - the Datasets across all BUILD Projects - - Workflows - the Workflows across all BUILD Projects - - Activities - activities overview and monitoring +- EXPLORE - for Knowledge Graph browsing and exploration, specifically + - [Knowledge Graphs](../explore-and-author/index.md) - a generic and extensible RDF data browser and editor + - [Vocabularies](../explore-and-author/vocabulary-catalog/index.md) - for vocabulary management + - [Thesauri](../explore-and-author/thesauri-management/index.md) - for managing thesauri and taxonomies based on SKOS + - [Queries](../explore-and-author/query-module/index.md) - a SPARQL query interface +- [BUILD](../build/index.md) - for creating and integrating Knowledge Graphs, with specific links to + - Projects - the BUILD Projects level + - Datasets - the Datasets across all BUILD Projects + - Workflows - the Workflows across all BUILD Projects + - Activities - activities overview and monitoring !!! note diff --git a/docs/getting-started/with-your-sandbox/index.md b/docs/getting-started/with-your-sandbox/index.md index 20ea4315..6fbe797c 100644 --- a/docs/getting-started/with-your-sandbox/index.md +++ b/docs/getting-started/with-your-sandbox/index.md @@ -17,7 +17,7 @@ eccenca's Corporate Memory is a platform for creating and managing Enterprise Kn
-- :material-rocket-launch:{ .lg .middle } __Get Started and Get Help__ +- :material-rocket-launch:{ .lg .middle } __Get Started and Get Help__ --- @@ -29,33 +29,33 @@ eccenca's Corporate Memory is a platform for creating and managing Enterprise Kn Find and contact us at: [:simple-github:](https://github.com/eccenca){target=_blank} • [:simple-x:](https://x.com/eccenca){target=_blank} • [:simple-linkedin:](https://de.linkedin.com/company/eccenca-gmbh){target=_blank} • [:octicons-mail-24:](mailto:info@eccenca.com) -- :fontawesome-brands-dropbox:{ .lg .middle } __Sandbox Resources__ +- :fontawesome-brands-dropbox:{ .lg .middle } __Sandbox Resources__ --- The sandbox includes a sample build project named _"Product Data Integration Demo"_ and the graphs generated by that project, as well as an integration graph as an entry point: _"Products - Integration"_. Shacl shapes are provided for the product vocabulary. These are used in the Business Knowledge Editor module for visual exploration as well as in a custom workspace configuration called _Product Data Integration_ to demonstrate how the user interface can be customized. -- :fontawesome-solid-graduation-cap:{ .lg .middle } __Masterclass Material__ +- :fontawesome-solid-graduation-cap:{ .lg .middle } __Masterclass Material__ --- - A list of materials and resources to reproduce and follow the masterclass session: _From Zero to KG Hero: Boosting Your KG Creation Productivity with eccenca Corporate Memory_. Originally presented at **The Knowledge Graph Conference 2023**. [Watch the recording on :simple-youtube:](https://youtu.be/qD-hge6gyIE){target=_blank}. + A list of materials and resources to reproduce and follow the masterclass session: _From Zero to KG Hero: Boosting Your KG Creation Productivity with eccenca Corporate Memory_. Originally presented at __The Knowledge Graph Conference 2023__. [Watch the recording on :simple-youtube:](https://youtu.be/qD-hge6gyIE){target=_blank}. [:octicons-arrow-right-24: materials and resources](./material.md){target=_blank} -- :material-script-text-play-outline:{ .lg .middle } __Tutorials and Examples__ +- :material-script-text-play-outline:{ .lg .middle } __Tutorials and Examples__ --- Our [tutorials](../../tutorials/index.md){target=_blank} help you to create Knowledge Graphs and to use the exploration and consumption features. To get started, we recommend: - - [Lift tabular sources (CSV, XSLX, JDBC)](../../build/lift-data-from-tabular-data-such-as-csv-xslx-or-database-tables/index.md){target=_blank} - - [Active Learning of Linking Rules](../../build/active-learning/index.md){target=_blank} - - [Building a customized User Interface](../../explore-and-author/graph-exploration/building-a-customized-user-interface/index.md){target=_blank} - - [Populate Data to Neo4j](../../consume/populate-data-to-neo4j/index.md){target=_blank} - - [Data in any Format via Custom API](../../consume/provide-data-in-any-format-via-a-custom-api/index.md){target=_blank} + - [Lift tabular sources (CSV, XSLX, JDBC)](../../build/lift-data-from-tabular-data-such-as-csv-xslx-or-database-tables/index.md){target=_blank} + - [Active Learning of Linking Rules](../../build/active-learning/index.md){target=_blank} + - [Building a customized User Interface](../../explore-and-author/graph-exploration/building-a-customized-user-interface/index.md){target=_blank} + - [Populate Data to Neo4j](../../consume/populate-data-to-neo4j/index.md){target=_blank} + - [Data in any Format via Custom API](../../consume/provide-data-in-any-format-via-a-custom-api/index.md){target=_blank} -- :material-graph:{ .lg .middle } __BUILD__ +- :material-graph:{ .lg .middle } __BUILD__ --- @@ -63,7 +63,7 @@ eccenca's Corporate Memory is a platform for creating and managing Enterprise Kn [:octicons-arrow-right-24: Learn more about _Build_](../../build/index.md){target=_blank} -- :material-compass-rose:{ .lg .middle } __EXPLORE__ +- :material-compass-rose:{ .lg .middle } __EXPLORE__ --- @@ -71,7 +71,7 @@ eccenca's Corporate Memory is a platform for creating and managing Enterprise Kn [:octicons-arrow-right-24: Learn more about _Explore_](../../explore-and-author/index.md){target=_blank} -- :material-api:{ .lg .middle } __CONSUME__ +- :material-api:{ .lg .middle } __CONSUME__ --- @@ -79,7 +79,7 @@ eccenca's Corporate Memory is a platform for creating and managing Enterprise Kn [:octicons-arrow-right-24: Learn more about _Consume_](../../consume/index.md){target=_blank} -- :other-powerautomate:{ .lg .middle } __AUTOMATE__ +- :other-powerautomate:{ .lg .middle } __AUTOMATE__ --- @@ -87,7 +87,7 @@ eccenca's Corporate Memory is a platform for creating and managing Enterprise Kn [:octicons-arrow-right-24: Learn more about _Automate_](../../automate/index.md){target=_blank} -- :material-account-school:{ .lg .middle } __Training and Certification__ +- :material-account-school:{ .lg .middle } __Training and Certification__ --- @@ -95,7 +95,7 @@ eccenca's Corporate Memory is a platform for creating and managing Enterprise Kn [:octicons-arrow-right-24: register at _eccenca LMS_](https://lms.eccenca.com/){target=_blank} -- :material-information-variant:{ .lg .middle } __About Corporate Memory__ +- :material-information-variant:{ .lg .middle } __About Corporate Memory__ --- diff --git a/docs/getting-started/with-your-sandbox/material.md b/docs/getting-started/with-your-sandbox/material.md index 8f63fbe9..144aba02 100644 --- a/docs/getting-started/with-your-sandbox/material.md +++ b/docs/getting-started/with-your-sandbox/material.md @@ -15,7 +15,7 @@ A list of materials and resources to reproduce and follow the masterclass (MC).
-- ## File resources +- ## File resources --- @@ -29,7 +29,7 @@ A list of materials and resources to reproduce and follow the masterclass (MC). *) vocabulary already installed, attached for information purposes only. -- ## Name(space) suggestions +- ## Name(space) suggestions --- @@ -43,7 +43,7 @@ A list of materials and resources to reproduce and follow the masterclass (MC). | Dataset (KG) | MC Prod - Links | `http://mc.eccenca.com/prod-links/` | | Build Project | MC Product Build Demo | | -- ## Resource IRI suggestions +- ## Resource IRI suggestions --- @@ -53,7 +53,7 @@ A list of materials and resources to reproduce and follow the masterclass (MC). | Employee | `http://mc.eccenca.com/prod-data/empl-{email}` | | Hardware | `http://mc.eccenca.com/prod-data/hw-{id}` | | Price | `http://mc.eccenca.com/prod-data/price-{parent-id}-{currency}` | - | Product Category | `http://mc.eccenca.com/prod-data/prod-cat-{name|uuid}` | + | Product Category | `http://mc.eccenca.com/prod-data/prod-cat-{uuid(name)}` | | Service | `http://mc.eccenca.com/prod-data/srv-{id}` | | Supplier | `http://mc.eccenca.com/prod-data/suppl-{id}` | diff --git a/docs/index.md b/docs/index.md index b3307ba2..66b037ba 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,59 +8,58 @@ hide:
-- :material-calendar: [Release Notes](release-notes/corporate-memory-24-3/index.md) +- :material-calendar: [Release Notes](release-notes/corporate-memory-24-3/index.md) --- Documentation of changes and enhancements for each version. -- :material-list-status: [Tutorials](tutorials/index.md) +- :material-list-status: [Tutorials](tutorials/index.md) --- Learn by example and step-by-step guidelines to achieve a concrete goal fast. -- :material-arrow-right-thick: [Getting Started](getting-started/index.md) +- :material-arrow-right-thick: [Getting Started](getting-started/index.md) --- This page describes how to work with Corporate Memory and shortly outlines all functionalities of the user interface. -- :material-star: [Build](build/index.md) +- :material-star: [Build](build/index.md) --- Lift your data by integrating multiple datasets into a Knowledge Graph. -- :material-star: [Explore and Author](explore-and-author/index.md) +- :material-star: [Explore and Author](explore-and-author/index.md) --- Explore, author and interact with your Knowledge Graph. -- :material-star: [Consume](consume/index.md) +- :material-star: [Consume](consume/index.md) --- This section outlines how to consume data from the Knowledge Graph. -- :material-star-outline: [Deploy and Configure](deploy-and-configure/index.md) +- :material-star-outline: [Deploy and Configure](deploy-and-configure/index.md) --- Deploy in your own environment. -- :material-star-outline: [Automate](automate/index.md) +- :material-star-outline: [Automate](automate/index.md) --- Setup processes and automate activities based on and towards your Knowledge Graph. -- :material-star-outline: [Develop](develop/index.md) +- :material-star-outline: [Develop](develop/index.md) --- API documentation and programming recipes.
- diff --git a/docs/release-notes/corporate-memory-19-10/index.md b/docs/release-notes/corporate-memory-19-10/index.md index 6e0935b7..345be7bc 100644 --- a/docs/release-notes/corporate-memory-19-10/index.md +++ b/docs/release-notes/corporate-memory-19-10/index.md @@ -80,56 +80,55 @@ In addition to that, these changes are shipped: In addition to that, multiple performance and stability issues were solved. - ## eccenca DataManager v19.10 This version of eccenca DataManager adds the following new features: -- New module `task` - - Offers a direct resource actions. Interfaces only available by URL. See documentation for more details. - - Path `/task/resource/create` allows to create a new resource by given graph and type. -- General - - Config parameter `js.config.api.defaultTimeout` for default UI queries timeout. - - Config parameter `js.config.resourceTable.timeoutDownload` for Resource Table timeout on download requests on Explore and Query modules. - - Validation of mandatory fields in `shacline` view. - - Add new property `shui:onUpdateUpdate` for `sh:NodeShape`. -- Module Explore - - Config parameter `js.config.modules.explore.graphlist.whiteList` to filter specific graphs. - - Config parameter `js.config.modules.explore.graphlist.internalGraphs` to hide specific graphs. - - Config parameter `js.config.modules.explore.navigation.itemsPerPage` show items per page in navigation box. - - Support for inverse property relations. -- Module Query - - Config parameter `js.config.modules.query.timeout` for manual queries. - - Config parameter `js.config.modules.query.graph` to define the graph were data is saved and requested. +- New module `task` + - Offers a direct resource actions. Interfaces only available by URL. See documentation for more details. + - Path `/task/resource/create` allows to create a new resource by given graph and type. +- General + - Config parameter `js.config.api.defaultTimeout` for default UI queries timeout. + - Config parameter `js.config.resourceTable.timeoutDownload` for Resource Table timeout on download requests on Explore and Query modules. + - Validation of mandatory fields in `shacline` view. + - Add new property `shui:onUpdateUpdate` for `sh:NodeShape`. +- Module Explore + - Config parameter `js.config.modules.explore.graphlist.whiteList` to filter specific graphs. + - Config parameter `js.config.modules.explore.graphlist.internalGraphs` to hide specific graphs. + - Config parameter `js.config.modules.explore.navigation.itemsPerPage` show items per page in navigation box. + - Support for inverse property relations. +- Module Query + - Config parameter `js.config.modules.query.timeout` for manual queries. + - Config parameter `js.config.modules.query.graph` to define the graph were data is saved and requested. In addition to that, these changes are shipped: -- General - - Default pagination size of 20 elements for all Resource Tables. - - Allow datatype `xsd:anyURI` for literals. - - Upgraded to react 16. -- Module Explore - - Merged graph view `RDFDoc` into 'resource details view'. - - Renamed global search label. - - Graph creation will add the type `void:Dataset` instead of `owl:Ontology`. - - Use the label of the type of the instances for the name of the CSV file downloaded from the Resource Table. - - Display the context graph in `properties` and `references` tables. -- Module Dataset - - Adjusted position and tooltip of parameter `uriProperty` in 'Add data stepper'. -- Module Query - - Use the dataset label for the name of the CSV file downloaded from the Resource Table. -- Module Login - - Renew tokens when they expire. -- Module Administration - - Allow to search in IRIs for list of readable and writeable graphs. +- General + - Default pagination size of 20 elements for all Resource Tables. + - Allow datatype `xsd:anyURI` for literals. + - Upgraded to react 16. +- Module Explore + - Merged graph view `RDFDoc` into 'resource details view'. + - Renamed global search label. + - Graph creation will add the type `void:Dataset` instead of `owl:Ontology`. + - Use the label of the type of the instances for the name of the CSV file downloaded from the Resource Table. + - Display the context graph in `properties` and `references` tables. +- Module Dataset + - Adjusted position and tooltip of parameter `uriProperty` in 'Add data stepper'. +- Module Query + - Use the dataset label for the name of the CSV file downloaded from the Resource Table. +- Module Login + - Renew tokens when they expire. +- Module Administration + - Allow to search in IRIs for list of readable and writeable graphs. The following features have been removed in this release: -- Module Explore - - Config parameter `js.config.modules.explore.graphlist.listQuery` which is now obsolete. - - Config parameter `js.config.modules.explore.details.history` which is now obsolete as the feature is no longer supported. - - 'History' tab. -- Module Sync also known as `SubscriptionManagement`. +- Module Explore + - Config parameter `js.config.modules.explore.graphlist.listQuery` which is now obsolete. + - Config parameter `js.config.modules.explore.details.history` which is now obsolete as the feature is no longer supported. + - 'History' tab. +- Module Sync also known as `SubscriptionManagement`. In addition to that, multiple stability issues were solved. @@ -137,34 +136,34 @@ In addition to that, multiple stability issues were solved. This version of eccenca DataPlatform adds the following new features: -- SPARQL 1.1 Query endpoint - - An `in-iris` property to the JSON `search` parameter to enable search over IRIs. - - A `timeout` parameter which allows to configure the maximal amount of milliseconds that a query execution can run. - - Support for Microsoft Excel (`.xlsx`) file download for `SELECT` queries. -- SPARQL 1.1 Update endpoint - - A `timeout` parameter which allows to configure the maximal amount of milliseconds that an update execution can run (Stardog only). -- SPARQL 1.1 Graph Store Protocol - - `multipart/form-data` support for HTTP PUT. - - Added the `timeout` parameter, which allows to configure the maximal amount of milliseconds that a request execution should run. - - Documentation for content negotiation by `format` query parameter. +- SPARQL 1.1 Query endpoint + - An `in-iris` property to the JSON `search` parameter to enable search over IRIs. + - A `timeout` parameter which allows to configure the maximal amount of milliseconds that a query execution can run. + - Support for Microsoft Excel (`.xlsx`) file download for `SELECT` queries. +- SPARQL 1.1 Update endpoint + - A `timeout` parameter which allows to configure the maximal amount of milliseconds that an update execution can run (Stardog only). +- SPARQL 1.1 Graph Store Protocol + - `multipart/form-data` support for HTTP PUT. + - Added the `timeout` parameter, which allows to configure the maximal amount of milliseconds that a request execution should run. + - Documentation for content negotiation by `format` query parameter. The following features have been removed in this release: -- Data Sharing: A WebSub based Publish-Subscribe service for RDF named graphs. -- IoT Permissions Plugin: A plugin which enables the usage of the IoT Permissions Service API 2. -- OAuth 2.0 authorization server: Issues access tokens to a client after successfully authenticating a user. -- Authentication: User management via authentication providers as it was only needed by the OAuth 2.0 authorization server. +- Data Sharing: A WebSub based Publish-Subscribe service for RDF named graphs. +- IoT Permissions Plugin: A plugin which enables the usage of the IoT Permissions Service API 2. +- OAuth 2.0 authorization server: Issues access tokens to a client after successfully authenticating a user. +- Authentication: User management via authentication providers as it was only needed by the OAuth 2.0 authorization server. In addition to that, these changes are shipped: -- Stardog - - Upgraded support to version 7.0.2. - - Versioning does no longer work with Stardog 7. - - Legacy versioning support for Stardog 6 (deprecated). -- OAuth 2.0: Resource protection is now mandatory (can no longer be disabled, use anonymous access instead). -- SPARQL 1.1 Query endpoint - - The value of the `string` property of the JSON `search` parameter is now tokenized which means that each token will be searched separately. Only results matching all tokens will be returned. - - Updated Spring Boot version from 1.5.21 to 1.5.22. +- Stardog + - Upgraded support to version 7.0.2. + - Versioning does no longer work with Stardog 7. + - Legacy versioning support for Stardog 6 (deprecated). +- OAuth 2.0: Resource protection is now mandatory (can no longer be disabled, use anonymous access instead). +- SPARQL 1.1 Query endpoint + - The value of the `string` property of the JSON `search` parameter is now tokenized which means that each token will be searched separately. Only results matching all tokens will be returned. + - Updated Spring Boot version from 1.5.21 to 1.5.22. In addition to that, multiple performance and stability issues were solved. @@ -172,16 +171,16 @@ In addition to that, multiple performance and stability issues were solved. With the removal of the OAuth 2.0 authorization server capability, many configuration properties have been changed. -- Removed - - The properties `oauth2.clients.*` have been removed. - - The properties `authentication.*` have been removed. -- Moved - - The property `oauth2.jwt.signing.verificationKey` has been moved to `security.oauth2.resource.jwt.keyValue` . - - The property `oauth2.anonymous` has been moved to `security.oauth2.resource.anonymous` . - - The claims mapping properties under `oauth2.resourceServer.claimsMapping.*` have been moved to `security.oauth2.resource.jwt.claims.*` . - - The properties `oauth2.authorizeRequests.*` to configure the resources to be protected by the resource server have been moved to `security.oauth2.resource.authorizeRequests.*` . -- Added - - The value of the property `security.oauth2.resource.id`  (defaults to `dataplatform`) must be part of the `aud` (audience) claim in the JWT used to access a protected resource. +- Removed + - The properties `oauth2.clients.*` have been removed. + - The properties `authentication.*` have been removed. +- Moved + - The property `oauth2.jwt.signing.verificationKey` has been moved to `security.oauth2.resource.jwt.keyValue` . + - The property `oauth2.anonymous` has been moved to `security.oauth2.resource.anonymous` . + - The claims mapping properties under `oauth2.resourceServer.claimsMapping.*` have been moved to `security.oauth2.resource.jwt.claims.*` . + - The properties `oauth2.authorizeRequests.*` to configure the resources to be protected by the resource server have been moved to `security.oauth2.resource.authorizeRequests.*` . +- Added + - The value of the property `security.oauth2.resource.id`  (defaults to `dataplatform`) must be part of the `aud` (audience) claim in the JWT used to access a protected resource. Don't forget to update your configuration accordingly. For instance, assuming you have the following old configuration: @@ -211,7 +210,6 @@ oauth2: key: 'groups' ``` - The migrated properties should look like this: ``` yaml @@ -229,4 +227,3 @@ security: groups: groups # optional, defaults to `groups` clientId: azp # optional, defaults to `azp` ``` - diff --git a/docs/release-notes/corporate-memory-20-03/index.md b/docs/release-notes/corporate-memory-20-03/index.md index 28b189c8..2e56e9a4 100644 --- a/docs/release-notes/corporate-memory-20-03/index.md +++ b/docs/release-notes/corporate-memory-20-03/index.md @@ -30,7 +30,6 @@ More detailed release notes for these versions are listed below. ## eccenca DataIntegration v20.03 - This version of eccenca DataIntegration adds the following new features: - Support for additional value types for mapping targets (XML Schema date/time types, duration, etc.). @@ -67,7 +66,6 @@ In addition to that, these changes are shipped: In addition to that, multiple performance and stability issues were solved. - ## eccenca DataManager v20.03 This version of eccenca DataManager adds the following new features: @@ -104,7 +102,6 @@ The following features have been removed in this release: In addition to that, multiple stability issues were solved. - ## eccenca DataPlatform v20.03 This version of eccenca DataPlatform adds the following new features: @@ -154,7 +151,6 @@ This version of eccenca Corporate Memory Control (cmemc) adds the following new - `workspace` command group, to `import` and `export` the workspace - ability to work with SSL enabled deployments (add CA certs) - ## Migration Notes ### DataIntegration @@ -166,7 +162,7 @@ With v20.03 the following changes need to be made in your dataintegration.conf f - both take an arbitrary alpha numerical string of minimum 16 characters length - depending on your deployment set them in your `production.conf` or `application.conf` DataIntegration configuration file -``` +```conf ... play.http.secret.key = "uiodshfoun78qwg8asd7gfasdasddfgn87gsn8fdsngasdfsngf8ds" ... @@ -222,4 +218,3 @@ spring: jwt: jwk-set-uri: http://keycloak:8080/auth/realms/cmem/protocol/openid-connect/certs ``` - diff --git a/docs/release-notes/corporate-memory-20-06/index.md b/docs/release-notes/corporate-memory-20-06/index.md index e3b273db..a6d70d57 100644 --- a/docs/release-notes/corporate-memory-20-06/index.md +++ b/docs/release-notes/corporate-memory-20-06/index.md @@ -194,4 +194,3 @@ With v20.06 the following changed need to be made: - `cmemc query execute --accept '*'` - `query list` has a different default output, to return to the previous behavior change your cmemc `query list` calls to: - `cmemc query list --id-only` - diff --git a/docs/release-notes/corporate-memory-20-10/index.md b/docs/release-notes/corporate-memory-20-10/index.md index b0c0ca7c..579256c8 100644 --- a/docs/release-notes/corporate-memory-20-10/index.md +++ b/docs/release-notes/corporate-memory-20-10/index.md @@ -29,7 +29,6 @@ This release delivers the following component versions: More detailed release notes for these versions are listed below. - ## eccenca DataIntegration v20.10.1 This version of eccenca DataIntegration adds the following new features: @@ -102,7 +101,6 @@ Finally, the following performance and stability issues were solved: - Deleting S3 backed resources broken due to a slash added to filenames - Update PostgreSQL driver to v42.2.14 because of security vulnerability - ## eccenca DataManager v20.10.1 This version of eccenca DataManager adds the following new features: @@ -148,7 +146,6 @@ In addition to that, these changes are shipped: In addition to that, multiple performance and stability issues were solved. - ## eccenca DataPlatform v20.10 This version of eccenca DataPlatform adds the following new features: @@ -173,7 +170,6 @@ This version of eccenca DataPlatform adds the following new features: In addition to that, multiple performance and stability issues were solved. - ## eccenca Corporate Memory Control (cmemc) v20.10 This version of cmemc adds the following new features: @@ -196,19 +192,18 @@ In addition to that, these changes are shipped: - The completion of `--filename-template` resulted in files with wrong chars. - The python version is disabled in completion mode. - ## eccenca Corporate Memory PowerBI Connector (v20.10) This release of our PowerBI Connector does not introduce new features or relevant changes. We provided a tutorial on how to use this component: [Consuming Graphs in Power BI](../../consume/consuming-graphs-in-power-bi/index.md) - # Migration Notes ## DataIntegration - XML serialization for meta data elements is not forward compatible, i.e. projects exported with this version cannot be imported in older DataIntegration versions. - The logout URL needs to be set to make sure that DataIntegration also triggers a logout inside the Keycloak instance: - ``` + + ```conf oauth.logoutRedirectUrl = ${DEPLOY_BASE_URL}"/auth/realms/cmem/protocol/openid-connect/logout?redirect_uri="${DEPLOY_BASE_URL} ``` @@ -219,6 +214,7 @@ This release of our PowerBI Connector does not introduce new features or relevan - The modules `build` as well as `datasets` are disabled now by default. - The module `explore` is the default first entry point (`startsWith`). - This section needs to be added to each workspace configuration: + ``` yaml DIWorkspace: enable: true @@ -228,4 +224,3 @@ This release of our PowerBI Connector does not introduce new features or relevan ## cmemc - If your automation scripts rely on the created file name of the project export command, you need to change your scripts and set the old export name explicitly with `-t {{id}}`. - diff --git a/docs/release-notes/corporate-memory-20-12/index.md b/docs/release-notes/corporate-memory-20-12/index.md index 52b48fd4..e46935de 100644 --- a/docs/release-notes/corporate-memory-20-12/index.md +++ b/docs/release-notes/corporate-memory-20-12/index.md @@ -152,10 +152,11 @@ We provided a tutorial on how to use this component: [Consuming Graphs in Power ### DataManager - In your workspaces configuration add `DIWorkspace.baseUrl` (mostly this will be `"/dataintegration"`): + ``` yaml js.config.workspaces: default: - ... + ... DIWorkspace: ... baseUrl: /dataintegration @@ -164,4 +165,3 @@ js.config.workspaces: ### cmemc - The `config check` command has been deprecated, please use the `admin status` command instead. - diff --git a/docs/release-notes/corporate-memory-21-02/index.md b/docs/release-notes/corporate-memory-21-02/index.md index d7eeaf02..23f2f19a 100644 --- a/docs/release-notes/corporate-memory-21-02/index.md +++ b/docs/release-notes/corporate-memory-21-02/index.md @@ -200,4 +200,3 @@ In addition to that, these changes are shipped: - `workflow list` command, use the `--id-only` or `CMEMC_WORKFLOW_LIST_ID_ONLY=true` to get the ID list. - `graph list` command, use `--filter access readonly`|`writeable` instead of `--filter` `readonly`|`writeable` - The command config check was removed (was deprecated in v20.12) - diff --git a/docs/release-notes/corporate-memory-21-04/index.md b/docs/release-notes/corporate-memory-21-04/index.md index 97e4f73d..64db2437 100644 --- a/docs/release-notes/corporate-memory-21-04/index.md +++ b/docs/release-notes/corporate-memory-21-04/index.md @@ -103,10 +103,10 @@ This version of eccenca DataManager adds the following new features: In addition to that, these changes are shipped: - General - - Use redux store to manage notifications in DataManager (MessageHandler) and improve error parse / handle - - Use redux store to manage main application state. - - Change value of `js.config.modules.explore.overallSearchQuery` and `js.config.modules.explore.navigation.searchQuery` to use the `""""` SPARQL string separator. - - *BREAK* please use `"""` if you use custom queries for that values + - Use redux store to manage notifications in DataManager (MessageHandler) and improve error parse / handle + - Use redux store to manage main application state. + - Change value of `js.config.modules.explore.overallSearchQuery` and `js.config.modules.explore.navigation.searchQuery` to use the `""""` SPARQL string separator. + - *BREAK* please use `"""` if you use custom queries for that values - Development - Switch to GUI elements repository from Github @@ -202,4 +202,3 @@ No migration notes - The exit code values of `project import` and `export` commands are fixed (in case of failure) so you may have to change these calls in your scripts. - The deprecated `workspace` command group is now only available as `admin workspace` command group so you have to change these calls in scripts. - diff --git a/docs/release-notes/corporate-memory-21-06/index.md b/docs/release-notes/corporate-memory-21-06/index.md index 302df023..aab44042 100644 --- a/docs/release-notes/corporate-memory-21-06/index.md +++ b/docs/release-notes/corporate-memory-21-06/index.md @@ -6,7 +6,6 @@ tags: Corporate Memory 21.06 is the third release in 2021. - ![21.06 Workflow Editor](21.06-WorkflowEditor.png "21.06 Workflow Editor") ![21.06 Vocabulary Viewer](21.06-VocabularyViewer.png "21.06 Vocabulary Viewer") @@ -160,7 +159,8 @@ In addition to that, these changes are shipped: - This does not influence the actual execution of the workflows in any way. - An auto-layouting feature will be added in the future - Plugin configuration has been changed. The 'plugin.blacklist' has been deprecated and will be removed in future versions. See example below for new format: -``` + +```conf pluginRegistry { # External plugins are loaded from this folder pluginFolder = ${elds.home}"/etc/dataintegration/plugins/" @@ -184,5 +184,3 @@ No migration notes ### cmemc - The configuration keys `*_ENTRY` are not supported anymore. In case you used them, switch to `*_PROCESS` configuration - - diff --git a/docs/release-notes/corporate-memory-21-11/index.md b/docs/release-notes/corporate-memory-21-11/index.md index 583aa1b4..9e314fd4 100644 --- a/docs/release-notes/corporate-memory-21-11/index.md +++ b/docs/release-notes/corporate-memory-21-11/index.md @@ -117,7 +117,7 @@ This version of eccenca DataManager adds the following new features: - Vocabs Module - Allow create new empty ontology without uploading a file. - Check if graph exist and show an error while creating a new vocab. -- Explore +- Explore - Allow hide / show the vocab viz module via configuration `details.visualization.enable` - Center automatically load vocab viz on load - Show precise tooltips for controls of vocab viz @@ -162,6 +162,7 @@ This version of eccenca DataPlatform ships the following new features: - Prometheus and Spring metrics endpoints are now exposed per default, i.e. `./actuator/prometheus` or `actuator/metrics` for list and, exemplarily, `./actuator/metrics/cache.size` for the metric of interest, see the [spring doc](https://docs.spring.io/spring-boot/docs/current/reference/html/actuator.html#actuator.metrics.endpoint) for more information. - you can deactivate them using the configuration properties in `application.yml` (or any other spring config) + ``` yaml title="application.yml" endpoint: prometheus: @@ -169,7 +170,9 @@ endpoint: metrics: enabled: false ``` + - Users roles need to match values of `authorization.abox.adminGroup` or `authorization.abox.metricsGroup` role definition for accessing those endpoints. `authorization.abox.metricsGroup` defaults to `metrics`, therefore in keycloak a user needs to `metrics` added as role, for example via a group and groupmapping. + - graphdb lucene index support - the index is used for example in the explore section to allow fast and userfriendly access - Graph List @@ -287,6 +290,7 @@ No migration notes - Jinja templates will no longer fail on unknown tokens. If this was used for signaling errors or fail-fast evaluation, this has to be implemented in regular conditional checks. - Virtuoso config requires adjustments, its HTTP port needs to be configured. - Please ensure, that the configured user has the same access rights in virtuoso via ODBC and HTTP + ``` yaml title="application.yml (old)" sparqlEndpoints: virtuoso: @@ -297,7 +301,9 @@ sparqlEndpoints: username: "dba" password: "dba" ``` + becomes + ``` yaml title="application.yml (new)" sparqlEndpoints: virtuoso: diff --git a/docs/release-notes/corporate-memory-22-1/index.md b/docs/release-notes/corporate-memory-22-1/index.md index f4862eb5..003a10ae 100644 --- a/docs/release-notes/corporate-memory-22-1/index.md +++ b/docs/release-notes/corporate-memory-22-1/index.md @@ -30,10 +30,10 @@ The highlights of this release are: This release delivers the following component versions: -* eccenca DataPlatform v22.1 -* eccenca DataIntegration v22.1 -* eccenca DataManager v22.1.1 -* eccenca Corporate Memory Control (cmemc) v22.1.1 +- eccenca DataPlatform v22.1 +- eccenca DataIntegration v22.1 +- eccenca DataManager v22.1.1 +- eccenca Corporate Memory Control (cmemc) v22.1.1 More detailed release notes for these versions are listed below. @@ -95,7 +95,6 @@ In addition to that, these changes are shipped: In addition to that, multiple performance and stability issues were solved. - ## eccenca DataManager v22.1.1 This version of eccenca DataManager adds the following new features: @@ -198,7 +197,6 @@ The following commands are deprecated: In addition to that, multiple performance and stability issues were solved. - ## Migration Notes ### DataIntegration @@ -233,4 +231,3 @@ In addition to that, multiple performance and stability issues were solved. - deprecated commands - `admin bootstrap`|`showcase` are deprecated - use `admin store bootstrap`|`showcase` instead - diff --git a/docs/release-notes/corporate-memory-22-2/index.md b/docs/release-notes/corporate-memory-22-2/index.md index cadafdd1..044580b4 100644 --- a/docs/release-notes/corporate-memory-22-2/index.md +++ b/docs/release-notes/corporate-memory-22-2/index.md @@ -12,13 +12,13 @@ Corporate Memory 22.2.3 is the third patch release in the 22.2 release line. The highlights of this release are: -- Build: - - The all new **Active** (Link) **Learning UI** - - Extended **Python Plugin SDK** -- Explore: - - New graph exploration module **EasyNav** -- Automate: - - Tag filter, better status monitoring and complete query management +- Build: + - The all new **Active** (Link) **Learning UI** + - Extended **Python Plugin SDK** +- Explore: + - New graph exploration module **EasyNav** +- Automate: + - Tag filter, better status monitoring and complete query management !!! warning @@ -30,10 +30,10 @@ The highlights of this release are: This release delivers the following component versions: -- eccenca DataPlatform v22.2.2 -- eccenca DataIntegration v22.2.1 -- eccenca DataManager v22.2.3 -- eccenca Corporate Memory Control (cmemc) v22.2 +- eccenca DataPlatform v22.2.2 +- eccenca DataIntegration v22.2.1 +- eccenca DataManager v22.2.3 +- eccenca Corporate Memory Control (cmemc) v22.2 More detailed release notes for these versions are listed below. @@ -41,65 +41,65 @@ More detailed release notes for these versions are listed below. v22.2.1 of eccenca DataIntegration adds the following new features: -- Rule and workflow editors: - - Support automatic scrolling when moving beyond the editor canvas borders on a all drag and edge connect/update operations. -- Added "sort words" transform operator, which sorts words in each value. +- Rule and workflow editors: + - Support automatic scrolling when moving beyond the editor canvas borders on a all drag and edge connect/update operations. +- Added "sort words" transform operator, which sorts words in each value. In addition to that, these changes are included in v22.2.1 of eccenca DataIntegration: -- Rule editors (linking, transform): - - On tab change do not remove the search text, instead select the text to easily overwrite it. - - Allow to search for input paths in the `All` tab. -- If a long-running workflow is executed manually, the same workflow can be started by a scheduler in the background. -- Executing workflows did not occupy a slot in the thread pool (i.e., unlimited workflows could be executed concurrently). -- Generating links could lead to a deadlock, if no slot in the thread pool is available. -- Entering an invalid URI as path input in the linking editor with a knowledge graph as input results in the rule being broken in the editor. -- Linking editor: Show the same property labels in the input path auto-completion as in the tab auto-completion. +- Rule editors (linking, transform): + - On tab change do not remove the search text, instead select the text to easily overwrite it. + - Allow to search for input paths in the `All` tab. +- If a long-running workflow is executed manually, the same workflow can be started by a scheduler in the background. +- Executing workflows did not occupy a slot in the thread pool (i.e., unlimited workflows could be executed concurrently). +- Generating links could lead to a deadlock, if no slot in the thread pool is available. +- Entering an invalid URI as path input in the linking editor with a knowledge graph as input results in the rule being broken in the editor. +- Linking editor: Show the same property labels in the input path auto-completion as in the tab auto-completion. v22.2 of eccenca DataIntegration adds the following new features: -- New active learning UI -- Python plugins: Added context objects that allow accessing context dependent functionalities, such as: - - The current OAuth token - - Updating the execution report (for workflows) - - DI version - - Current project and task identifiers - - Requires `cmem-plugin-base >=2.0.0` -- Workflows search link in main navigation -- Linking rule editor - - Advanced parameter toggle that shows/hides advanced parameters like `weight` and advanced section in rule parameter modal -- Support for sticky notes in both linking and workflow editors -- Parameter `profiling.defaults.noEntities` to configure the default entity limit for profiling operations -- Parameter `org.silkframework.runtime.activity.concurrentExecutions` to set the max. concurrent activity instances -- Support for the `URI attribute` parameter of datasets -- Support for auto-configuration in create/update dialog -- Config parameters: - - `profiling.defaults.noEntities` to configure the default entity limit for profiling operations - - `org.silkframework.runtime.activity.concurrentExecutions` to set the max. concurrent activity instances - - `cors.enabled`, `cors.config.allowOrigins` and `cors.config.allowCredentials` to configure CORS settings +- New active learning UI +- Python plugins: Added context objects that allow accessing context dependent functionalities, such as: + - The current OAuth token + - Updating the execution report (for workflows) + - DI version + - Current project and task identifiers + - Requires `cmem-plugin-base >=2.0.0` +- Workflows search link in main navigation +- Linking rule editor + - Advanced parameter toggle that shows/hides advanced parameters like `weight` and advanced section in rule parameter modal +- Support for sticky notes in both linking and workflow editors +- Parameter `profiling.defaults.noEntities` to configure the default entity limit for profiling operations +- Parameter `org.silkframework.runtime.activity.concurrentExecutions` to set the max. concurrent activity instances +- Support for the `URI attribute` parameter of datasets +- Support for auto-configuration in create/update dialog +- Config parameters: + - `profiling.defaults.noEntities` to configure the default entity limit for profiling operations + - `org.silkframework.runtime.activity.concurrentExecutions` to set the max. concurrent activity instances + - `cors.enabled`, `cors.config.allowOrigins` and `cors.config.allowCredentials` to configure CORS settings In addition to that, these changes are included in v22.2: -- Move `outputTemplate` parameter to advanced section of XML dataset plugin -- Improved performance of conversions to floating point numbers -- Improved linking performance -- Show report on linking execution tab -- When the evaluation fails because of missing paths in the cache give specific error message with node highlighting instead of generic error notification -- Errors in invalid Python packages are recorded and returned, instead of failing -- Size of the activity thread pool can be configured -- Linking rule editor - - Show linking rule label above toolbar when in integrated mode - - Handle "reversible" comparators, e.g. "Greater than", by allowing to switch source/target inputs instead of setting the 'reverse' parameter -- DataPlatform API timeout is configurable now -- Workflow progress information was moved to node footer that is displayed empty when no information is available -- Docker image base: `debian:bullseye-20220912-slim` -- Return 503 error before exceeding the concurrent activity execution limit instead of discarding a running activity instance -- Do not execute empty object mapping rules to improve performance -- Remove root (start) page: - - Redirect to workbench project search page - - Remove legacy workspace link from user menu - - Add "load example project" action to user menu -- Show activity labels instead of IDs in task activity overview +- Move `outputTemplate` parameter to advanced section of XML dataset plugin +- Improved performance of conversions to floating point numbers +- Improved linking performance +- Show report on linking execution tab +- When the evaluation fails because of missing paths in the cache give specific error message with node highlighting instead of generic error notification +- Errors in invalid Python packages are recorded and returned, instead of failing +- Size of the activity thread pool can be configured +- Linking rule editor + - Show linking rule label above toolbar when in integrated mode + - Handle "reversible" comparators, e.g. "Greater than", by allowing to switch source/target inputs instead of setting the 'reverse' parameter +- DataPlatform API timeout is configurable now +- Workflow progress information was moved to node footer that is displayed empty when no information is available +- Docker image base: `debian:bullseye-20220912-slim` +- Return 503 error before exceeding the concurrent activity execution limit instead of discarding a running activity instance +- Do not execute empty object mapping rules to improve performance +- Remove root (start) page: + - Redirect to workbench project search page + - Remove legacy workspace link from user menu + - Add "load example project" action to user menu +- Show activity labels instead of IDs in task activity overview In addition to that, multiple performance and stability issues are addressed. @@ -107,46 +107,46 @@ In addition to that, multiple performance and stability issues are addressed. v22.2.3 of eccenca DataManager has the following fixes: -- LinkRules +- LinkRules - Rule Setup: Fix display of filter v22.2.2 of eccenca DataManager has the following fixes: -- General +- General - Logout in DM also triggers logout in DI -- LinkRules +- LinkRules - Rule Setup: Rule filter correctly displays OneOf and NoneOf - Rule is correctly serialized after editing, preventing the rule contents to be deleted v22.2.1 of eccenca DataManager has the following fixes: -- LinkRules - - Fixed trigger of refetching data after an update - - Display of negative Reference Links +- LinkRules + - Fixed trigger of refetching data after an update + - Display of negative Reference Links v22.2 of eccenca DataManager adds the following new features: -- Navigation - - Add DataIntegration workflows link to main navigation -- Vocabulary Catalog - - Inline vocabulary metadata via (editable) shape - - Ability to activate git synchronization of changes - - Change history with diff view and ability to revert to a specific commit -- Explore - - New (Shacl) Template based graph creation wizard - - Supporting different methods to define / select graph IRIs - - Support for bulk add via `.zip` archives containing multiple RDF files -- i18n - - French translation -- EasyNav - - New graph visualization module - - With search filter configuration - - Bulk node search and bulk add - - Ability to save, load and share explorations +- Navigation + - Add DataIntegration workflows link to main navigation +- Vocabulary Catalog + - Inline vocabulary metadata via (editable) shape + - Ability to activate git synchronization of changes + - Change history with diff view and ability to revert to a specific commit +- Explore + - New (Shacl) Template based graph creation wizard + - Supporting different methods to define / select graph IRIs + - Support for bulk add via `.zip` archives containing multiple RDF files +- i18n + - French translation +- EasyNav + - New graph visualization module + - With search filter configuration + - Bulk node search and bulk add + - Ability to save, load and share explorations In addition to that, these changes are included in v22.2 of eccenca DataManager: -- Increase height of Turtle editor in the resource details view. +- Increase height of Turtle editor in the resource details view. In addition to that, multiple performance and stability issues are addressed. @@ -154,87 +154,87 @@ In addition to that, multiple performance and stability issues are addressed. v22.2.2 of eccenca DataPlatform has the following changes: -- Fixed +- Fixed - reintroduced support for IRI templates in node shapes, with only the uuid placeholder. - Prevent buffer overflow for large query results streaming to client -- Changed +- Changed - Maintenance: Updated Spring Boot to 2.7.8 v22.2.1 of eccenca DataPlatform has the following fixes: -- Update of dependencies because of vulnerabilities i.e. Spring Boot. -- Addition of logstash runtime dependency as to enable json logging. -- GraphDb indices are created without facet option causing problems. -- Fix of memory leak in query monitor causing high heap usage. -- Refactoring of spring integration tests (IT) and inclusion of most tests in the cucumber subproject. +- Update of dependencies because of vulnerabilities i.e. Spring Boot. +- Addition of logstash runtime dependency as to enable json logging. +- GraphDb indices are created without facet option causing problems. +- Fix of memory leak in query monitor causing high heap usage. +- Refactoring of spring integration tests (IT) and inclusion of most tests in the cucumber subproject. v22.2 of eccenca DataPlatform ships the following new features: -- Added support for manual query/update cancellation: - - active for graphdb, stardog, neptune - - DELETE `/api/admin/currentQueries/{queryId}` - - Neptune updates cannot be cancelled because queryId header not processed -- Added support for creation of configured graphdb repository on DP startup - - `store.graphdb.createRepositoryOnStartup`: Flag if repository shall be created on startup (default: false) -- Added support for selective invalidation of caches (graph list, shapes) via Update parsing / GraphDb Change Tracking - - `proxy.cache-selective-invalidation`: true if activated, false otherwise full flush on every write (default: true) - - `store.graphdb.graphDbChangeTrackingActive`: Whether change tracking for updates is active - better results for cache invalidation (default: true) - - `store.graphdb.graphDbChangeTrackingMaxQuadMemory`: Amount of quads as a result of an update which are loaded into memory for analyzing consequences for caches (default: 1000) -- Automatic creation of default `application.yml` and gradle tasks for generation of markdown documentation -- Added endpoints for supporting easynav graph visualizations - - search and resource listing via `/api/search` - - managing of persisted visualisations via `/api/navigate` endpoints -- Added provisioning of jinja templates with provided substitution map for endpoint `/api/custom/{slug}` -- Added property `proxy.descriptionProperties` (analogous to `proxy.labelProperties`) for defining search relevant description properties -- Extend query monitor - - Added fields per entry - - `timeout`: value in ms of the query/update timeout - - `timedOut`: boolean value on whether the query timed out or not - - `cancelled`: boolean value on whether the query has been cancelled manually - - `running`: boolean value on whether the query is currently still being executed - - `affectedGraphs`: on successfully finished query/update the affected graphs are shown (if possible to determine) - - Added property for memory bound consumption in MB for query monitor list - - `proxy.queryMonitorMaxMemoryInMb` (Default: 30) - - Added fields to prometheus metrics endpoint - - `querymonitor_memoryusage_total`: memory usage of query queue in MB - - `querymonitor_queuesize_total`: query queue size -- Extend actuator info endpoint with store backend properties, `/actuator/info`: - - fields under store: - - `type`: same as `store.type` property (MEMORY, HTTP, GRAPHDB, STARDOG, VIRTUOSO, NEPTUNE) - - `version`: if possible / otherwise UNKNOWN - - `host`: if applicable otherwise N/A - - `repository`: if applicable otherwise N/A - - `user`: if applicable otherwise N/A -- Add non-transactional git sync of graph changes - - graphs can be configured via graph configuration for bi-directional git sync - - cf. config properties under `gitSync.*` +- Added support for manual query/update cancellation: + - active for graphdb, stardog, neptune + - DELETE `/api/admin/currentQueries/{queryId}` + - Neptune updates cannot be cancelled because queryId header not processed +- Added support for creation of configured graphdb repository on DP startup + - `store.graphdb.createRepositoryOnStartup`: Flag if repository shall be created on startup (default: false) +- Added support for selective invalidation of caches (graph list, shapes) via Update parsing / GraphDb Change Tracking + - `proxy.cache-selective-invalidation`: true if activated, false otherwise full flush on every write (default: true) + - `store.graphdb.graphDbChangeTrackingActive`: Whether change tracking for updates is active - better results for cache invalidation (default: true) + - `store.graphdb.graphDbChangeTrackingMaxQuadMemory`: Amount of quads as a result of an update which are loaded into memory for analyzing consequences for caches (default: 1000) +- Automatic creation of default `application.yml` and gradle tasks for generation of markdown documentation +- Added endpoints for supporting easynav graph visualizations + - search and resource listing via `/api/search` + - managing of persisted visualisations via `/api/navigate` endpoints +- Added provisioning of jinja templates with provided substitution map for endpoint `/api/custom/{slug}` +- Added property `proxy.descriptionProperties` (analogous to `proxy.labelProperties`) for defining search relevant description properties +- Extend query monitor + - Added fields per entry + - `timeout`: value in ms of the query/update timeout + - `timedOut`: boolean value on whether the query timed out or not + - `cancelled`: boolean value on whether the query has been cancelled manually + - `running`: boolean value on whether the query is currently still being executed + - `affectedGraphs`: on successfully finished query/update the affected graphs are shown (if possible to determine) + - Added property for memory bound consumption in MB for query monitor list + - `proxy.queryMonitorMaxMemoryInMb` (Default: 30) + - Added fields to prometheus metrics endpoint + - `querymonitor_memoryusage_total`: memory usage of query queue in MB + - `querymonitor_queuesize_total`: query queue size +- Extend actuator info endpoint with store backend properties, `/actuator/info`: + - fields under store: + - `type`: same as `store.type` property (MEMORY, HTTP, GRAPHDB, STARDOG, VIRTUOSO, NEPTUNE) + - `version`: if possible / otherwise UNKNOWN + - `host`: if applicable otherwise N/A + - `repository`: if applicable otherwise N/A + - `user`: if applicable otherwise N/A +- Add non-transactional git sync of graph changes + - graphs can be configured via graph configuration for bi-directional git sync + - cf. config properties under `gitSync.*` In addition to that, these changes and fixes are included in v22.2 of eccence DataPlatform: -- New store configuration properties, see below for migration notes -- Changed property for defining select query for graphList - - setting is store dependant and not valid for some stores - - property `proxy.graphListQuery` (`proxy.graph_list_query`) moved to store settings: - - `store.stardog.graphListQuery` - - `store.neptune.graphListQuery` -- Changed property for scheduled cache invalidation - - `proxy.cacheInvalidationCron`: Spring boot cron entry cf. (default: `* */30 * * * *`) - - [https://docs.spring.io/spring-framework/docs/current/reference/html/integration.html#scheduling-cron-expression](https://docs.spring.io/spring-framework/docs/current/reference/html/integration.html#scheduling-cron-expression) -- Library updates including Spring Boot / Stardog -- Changed property for DP query system timeout - - `proxy.queryTimeoutGeneral` -> `store.queryTimeoutGeneral` in ISO 8601 duration format (default: `PT1H`) -- Changed loading of model entities i.e. shapes cache - - load model entities using GSP requests instead of construct queries - - Changed property for base IRI: `files.defaultBaseIri` to `proxy.defaultBaseIri` (default: `http://localhost/`) +- New store configuration properties, see below for migration notes +- Changed property for defining select query for graphList + - setting is store dependant and not valid for some stores + - property `proxy.graphListQuery` (`proxy.graph_list_query`) moved to store settings: + - `store.stardog.graphListQuery` + - `store.neptune.graphListQuery` +- Changed property for scheduled cache invalidation + - `proxy.cacheInvalidationCron`: Spring boot cron entry cf. (default: `* */30 * * * *`) + - [https://docs.spring.io/spring-framework/docs/current/reference/html/integration.html#scheduling-cron-expression](https://docs.spring.io/spring-framework/docs/current/reference/html/integration.html#scheduling-cron-expression) +- Library updates including Spring Boot / Stardog +- Changed property for DP query system timeout + - `proxy.queryTimeoutGeneral` -> `store.queryTimeoutGeneral` in ISO 8601 duration format (default: `PT1H`) +- Changed loading of model entities i.e. shapes cache + - load model entities using GSP requests instead of construct queries + - Changed property for base IRI: `files.defaultBaseIri` to `proxy.defaultBaseIri` (default: `http://localhost/`) The following functionalities have been discontinued: -- Support for provisioned store authorization -- Command line options create-config, update-war -- WAR build target and support for WAR servlet deployment -- Property for DP query system timeout check interval - - `proxy.queryTimeoutCheckCron` not necessary anymore -- Support for multiple endpoints +- Support for provisioned store authorization +- Command line options create-config, update-war +- WAR build target and support for WAR servlet deployment +- Property for DP query system timeout check interval + - `proxy.queryTimeoutCheckCron` not necessary anymore +- Support for multiple endpoints In addition to that, multiple performance and stability issues are addressed. @@ -242,75 +242,75 @@ In addition to that, multiple performance and stability issues are addressed. This version of cmemc adds the following new features: -- `project reload` command - - Reload all tasks of a project from the workspace provider -- `admin workspace python list-plugins` command - - New option `--package-id-only` to output only package IDs -- `admin workspace python install` command completion - - now also provides plugin packages published on pypi.org -- `query status` command - - New filter `query`: - - `graph` - List only queries which affected a certain graph (URL) - - `regex` - List only queries which query text matches a regular expression - - `trace-id` - List only queries which have the specified trace ID - - `user` - List only queries executed by the specified account (URL) - - New values for filter `status`: - - `cancelled`: List only queries which were cancelled - - `timeout`: List only queries which ran into a timeout -- `query cancel` command - - cancel a running query - this stops the execution in the backend - - Depending on the backend store, this will result in a broken result stream (stardog, neptune and virtuoso) or a valid result stream with incomplete results (graphdb) -- `dataset list`|`delete` commands - - New option `--filter` with the following concrete filter - - `project` - filter by project ID - - `regex` - filter by regular expression on the dataset label - - `tag` - filter by tag label - - `type` - filter by dataset type -- `workflow list` command - - New option `--filter` with the following concrete filter - - `project` - filter by project ID - - `regex` - filter by regular expression on the dataset label - - `tag` - filter by tag label - - `io` - filter by io type -- `admin status` command - - overall rewrite - - new table output - - new option `--raw` to output collected status / info values - - new option `--key` to output only specific values - - new option `--enforce-table` to enforce table output of `--key` -- `vocabular import` command - - new option `--namespace`: In case the imported vocabulary file does not include a preferred namespace prefix, you can manually add a namespace prefix -- `workflow io` command - - new flag `--autoconfig` / `--no-autoconfig` for input dataset auto configuration +- `project reload` command + - Reload all tasks of a project from the workspace provider +- `admin workspace python list-plugins` command + - New option `--package-id-only` to output only package IDs +- `admin workspace python install` command completion + - now also provides plugin packages published on pypi.org +- `query status` command + - New filter `query`: + - `graph` - List only queries which affected a certain graph (URL) + - `regex` - List only queries which query text matches a regular expression + - `trace-id` - List only queries which have the specified trace ID + - `user` - List only queries executed by the specified account (URL) + - New values for filter `status`: + - `cancelled`: List only queries which were cancelled + - `timeout`: List only queries which ran into a timeout +- `query cancel` command + - cancel a running query - this stops the execution in the backend + - Depending on the backend store, this will result in a broken result stream (stardog, neptune and virtuoso) or a valid result stream with incomplete results (graphdb) +- `dataset list`|`delete` commands + - New option `--filter` with the following concrete filter + - `project` - filter by project ID + - `regex` - filter by regular expression on the dataset label + - `tag` - filter by tag label + - `type` - filter by dataset type +- `workflow list` command + - New option `--filter` with the following concrete filter + - `project` - filter by project ID + - `regex` - filter by regular expression on the dataset label + - `tag` - filter by tag label + - `io` - filter by io type +- `admin status` command + - overall rewrite + - new table output + - new option `--raw` to output collected status / info values + - new option `--key` to output only specific values + - new option `--enforce-table` to enforce table output of `--key` +- `vocabular import` command + - new option `--namespace`: In case the imported vocabulary file does not include a preferred namespace prefix, you can manually add a namespace prefix +- `workflow io` command + - new flag `--autoconfig` / `--no-autoconfig` for input dataset auto configuration In addition to that, these changes and fixes are included: -- `admin workspace python list-plugins` command - - Additionally outputs the Package ID -- `project import` command - - The project id is now optional when importing project files -- `admin status` command - - new table output (similar to the other tables) - - `status` filter with `error` value - - only execution errors are listed - - this specifically means no cancelled and timeouted queries (they have there own status now) -- Add pysocks dependency to cmempy - - This allows for using the `all_proxy` evironment variable -- `dataset list --raw` output - - output was not a JSON array and not filtered correctly -- cmempy get graph streams - - stream enabled -- `admin status` command - - command will now always return, even if a component is down +- `admin workspace python list-plugins` command + - Additionally outputs the Package ID +- `project import` command + - The project id is now optional when importing project files +- `admin status` command + - new table output (similar to the other tables) + - `status` filter with `error` value + - only execution errors are listed + - this specifically means no cancelled and timeouted queries (they have there own status now) +- Add pysocks dependency to cmempy + - This allows for using the `all_proxy` evironment variable +- `dataset list --raw` output + - output was not a JSON array and not filtered correctly +- cmempy get graph streams + - stream enabled +- `admin status` command + - command will now always return, even if a component is down The following commands are discontinued: -- `admin bootstap` command - - was deprecated in 22.1, use `admin store bootstrap` command instead -- `admin showcase` command - - was deprecated in 22.1, use `admin store showcase` command instead -- `dataset list`|`delete` command - - `--project` option, use `--filter projext XXX` instead +- `admin bootstap` command + - was deprecated in 22.1, use `admin store bootstrap` command instead +- `admin showcase` command + - was deprecated in 22.1, use `admin store showcase` command instead +- `dataset list`|`delete` command + - `--project` option, use `--filter projext XXX` instead In addition to that, multiple performance and stability issues are addressed. @@ -326,7 +326,7 @@ In addition to that, multiple performance and stability issues are addressed. ### DataIntegration -- CSV attributes specified via the `properties` parameter had inconsistent encoding rules. For CSV datasets where the `properties` parameter is used this can lead to changed source paths. +- CSV attributes specified via the `properties` parameter had inconsistent encoding rules. For CSV datasets where the `properties` parameter is used this can lead to changed source paths. #### Python plugins @@ -334,78 +334,78 @@ Due to the added context classes, the signature of a number of functions has bee ##### WorkflowPlugin -- The execute function has a new parameter `context`: - - `def execute(self, inputs: Sequence[Entities], context: ExecutionContext)` +- The execute function has a new parameter `context`: + - `def execute(self, inputs: Sequence[Entities], context: ExecutionContext)` ##### ParameterType -- The `project_id` parameters of the label and the autocompletion functions have been replaced by the PluginContext: - - `def autocomplete(self, query_terms: list[str], context: PluginContext) -> list[Autocompletion]` - - `def label(self, value: str, context: PluginContext) -> Optional[str]` - - The project identifier can still be accessed via `context.project_id` -- The `fromString` function has a new parameter `context`: - - `def from_string(self, value: str, context: PluginContext) -> T` +- The `project_id` parameters of the label and the autocompletion functions have been replaced by the PluginContext: + - `def autocomplete(self, query_terms: list[str], context: PluginContext) -> list[Autocompletion]` + - `def label(self, value: str, context: PluginContext) -> Optional[str]` + - The project identifier can still be accessed via `context.project_id` +- The `fromString` function has a new parameter `context`: + - `def from_string(self, value: str, context: PluginContext) -> T` ### DataPlatform Due to the removed multiple endpoint support the store configuration properties have changed. Please revise your store configuration section(s) in your DataPlatform `application.yml`. The new configuration properties are: -- Type of store (general settings) - - `store.type`: MEMORY, HTTP, GRAPHDB, STARDOG, VIRTUOSO, NEPTUNE - - `store.authorization`: NONE, REWRITE_FROM -- MEMORY: - - `store.memory.files`: List of files loaded on startup -- HTTP: - - `store.http.queryEndpointUrl`: SPARQL Query endpoint (mandatory) - - `store.http.updateEndpointUrl`: SPARQL Update endpoint (mandatory) - - `store.http.graphStoreEndpointUrl`: SPARQL GSP endpoint (optional but highly recommended) - - `store.http.username`: Username (optional) - - `store.http.password`: Password (optional) -- GRAPHDB: - - `store.graphdb.host`: host of graphdb backend (i.e. localhost) - - `store.graphdb.port`: port of graphdb backend (i.e. 7200) - - `store.graphdb.ssl-enabled`: flag if ssl (https) is enabled (default: false) - - `store.graphdb.repository`: name of repository (i.e. cmem) - - `store.graphdb.username`: Username (optional) - - `store.graphdb.password`: Password (optional) - - `store.graphdb.useDirectTransfer`: flag if direct GSP endpoints of graphdb shall be used instead of workbench upload (default: true) - - `store.graphdb.importDirectory`: Import directory to be utilized in the "workbench import with shared folder" approach. - - `store.graphdb.graphDbChangeTrackingActive`: Whether change tracking for updates is active - better results for cache invalidation (default: true) - - `store.graphdb.graphDbChangeTrackingMaxQuadMemory`: Amount of quads as a result of an update which are loaded into memory for analyzing consequences for caches (default: 1000) -- STARDOG: - - `store.stardog.host`: host of stardog backend (i.e. localhost) - - `store.stardog.port`: port of stardog backend (i.e. 5820) - - `store.stardog.ssl-enabled`: flag if ssl (https) is enabled (default: false) - - `store.stardog.repository`: name of repository (i.e. cmem) - - `store.stardog.username`: Username (optional) - - `store.stardog.password`: Password (optional) - - `store.stardog.userPasswordSalt`: salt for generated user password (optional) - - `store.stardog.updateTimeoutInMilliseconds`: Timeout in ms for updates (default: 0 = deactivated) - - `store.stardog.graphListQuery`: Query for graph list - graph must be bound to variable ?g -- NEPTUNE: - - `store.neptune.host`: host of neptune backend (i.e. neptune-cluster123.eu-central-1.neptune.amazonaws.com) - - `store.neptune.port`: port of neptune backend (i.e. 8182) - - `store.neptune.graphListQuery`: Query for graph list - graph must be bound to variable ?g - - Settings under store.neptune.aws (mandatory): - - `store.neptune.aws.region`: AWS region where the configured neptune cluster is located (e.g. eu-central-1) - - `store.neptune.aws.authEnabled`: Flag on whether authentication is enabled on neptune cluster (default: true) - - Settings under `store.neptune.s3` for upload of large files (>150MB uncompressed) (optional): - - `store.neptune.s3.bucketNameOrAPAlias`: Name of bucket or access point for S3 bulk load - - `store.neptune.s3.iamRoleArn`: ARN of role under which neptune cluster loads from S3 - - `store.neptune.s3.bulkLoadThresholdInMb`: Load threshold in MB for GSP access, if graph data greater than S3 upload is used (default: 150) - - `store.neptune.s3.bulkLoadParallelism`: Degree of parallelism for neptune S3 bulk loader (LOW (default), MEDIUM, HIGH, OVERSUBSCRIBE) -- VIRTUOSO: - - `store.virtuoso.host`: host of virtuoso backend (i.e. localhost) - - `store.virtuoso.port`: http port of virtuoso backend (i.e. 8080) - - `store.virtuoso.databasePort`: database port of virtuoso backend (i.e. 1111) - - `store.virtuoso.ssl-enabled`: flag if ssl (https) is enabled (default: false) - - `store.virtuoso.username`: Username (optional) - - `store.virtuoso.password`: Password (optional) +- Type of store (general settings) + - `store.type`: MEMORY, HTTP, GRAPHDB, STARDOG, VIRTUOSO, NEPTUNE + - `store.authorization`: NONE, REWRITE_FROM +- MEMORY: + - `store.memory.files`: List of files loaded on startup +- HTTP: + - `store.http.queryEndpointUrl`: SPARQL Query endpoint (mandatory) + - `store.http.updateEndpointUrl`: SPARQL Update endpoint (mandatory) + - `store.http.graphStoreEndpointUrl`: SPARQL GSP endpoint (optional but highly recommended) + - `store.http.username`: Username (optional) + - `store.http.password`: Password (optional) +- GRAPHDB: + - `store.graphdb.host`: host of graphdb backend (i.e. localhost) + - `store.graphdb.port`: port of graphdb backend (i.e. 7200) + - `store.graphdb.ssl-enabled`: flag if ssl (https) is enabled (default: false) + - `store.graphdb.repository`: name of repository (i.e. cmem) + - `store.graphdb.username`: Username (optional) + - `store.graphdb.password`: Password (optional) + - `store.graphdb.useDirectTransfer`: flag if direct GSP endpoints of graphdb shall be used instead of workbench upload (default: true) + - `store.graphdb.importDirectory`: Import directory to be utilized in the "workbench import with shared folder" approach. + - `store.graphdb.graphDbChangeTrackingActive`: Whether change tracking for updates is active - better results for cache invalidation (default: true) + - `store.graphdb.graphDbChangeTrackingMaxQuadMemory`: Amount of quads as a result of an update which are loaded into memory for analyzing consequences for caches (default: 1000) +- STARDOG: + - `store.stardog.host`: host of stardog backend (i.e. localhost) + - `store.stardog.port`: port of stardog backend (i.e. 5820) + - `store.stardog.ssl-enabled`: flag if ssl (https) is enabled (default: false) + - `store.stardog.repository`: name of repository (i.e. cmem) + - `store.stardog.username`: Username (optional) + - `store.stardog.password`: Password (optional) + - `store.stardog.userPasswordSalt`: salt for generated user password (optional) + - `store.stardog.updateTimeoutInMilliseconds`: Timeout in ms for updates (default: 0 = deactivated) + - `store.stardog.graphListQuery`: Query for graph list - graph must be bound to variable ?g +- NEPTUNE: + - `store.neptune.host`: host of neptune backend (i.e. neptune-cluster123.eu-central-1.neptune.amazonaws.com) + - `store.neptune.port`: port of neptune backend (i.e. 8182) + - `store.neptune.graphListQuery`: Query for graph list - graph must be bound to variable ?g + - Settings under store.neptune.aws (mandatory): + - `store.neptune.aws.region`: AWS region where the configured neptune cluster is located (e.g. eu-central-1) + - `store.neptune.aws.authEnabled`: Flag on whether authentication is enabled on neptune cluster (default: true) + - Settings under `store.neptune.s3` for upload of large files (>150MB uncompressed) (optional): + - `store.neptune.s3.bucketNameOrAPAlias`: Name of bucket or access point for S3 bulk load + - `store.neptune.s3.iamRoleArn`: ARN of role under which neptune cluster loads from S3 + - `store.neptune.s3.bulkLoadThresholdInMb`: Load threshold in MB for GSP access, if graph data greater than S3 upload is used (default: 150) + - `store.neptune.s3.bulkLoadParallelism`: Degree of parallelism for neptune S3 bulk loader (LOW (default), MEDIUM, HIGH, OVERSUBSCRIBE) +- VIRTUOSO: + - `store.virtuoso.host`: host of virtuoso backend (i.e. localhost) + - `store.virtuoso.port`: http port of virtuoso backend (i.e. 8080) + - `store.virtuoso.databasePort`: database port of virtuoso backend (i.e. 1111) + - `store.virtuoso.ssl-enabled`: flag if ssl (https) is enabled (default: false) + - `store.virtuoso.username`: Username (optional) + - `store.virtuoso.password`: Password (optional) ### cmemc -- `dataset list`|`delete command` - - option `--project` is removed - - Please use `--filter project XXX` instead -- `admin status` command - - in case you piped the normal output of this command and reacted on that, you need to use the `--key` command now +- `dataset list`|`delete command` + - option `--project` is removed + - Please use `--filter project XXX` instead +- `admin status` command + - in case you piped the normal output of this command and reacted on that, you need to use the `--key` command now diff --git a/docs/release-notes/corporate-memory-23-1/index.md b/docs/release-notes/corporate-memory-23-1/index.md index 405e1429..a8eaaee6 100644 --- a/docs/release-notes/corporate-memory-23-1/index.md +++ b/docs/release-notes/corporate-memory-23-1/index.md @@ -12,14 +12,14 @@ Corporate Memory 23.1.3 is the second patch release in the 23.1 release line. The highlights of this release are: -- Build: - - Support for **global variables** in dataset and task parameters. - - Extensions to the Python Plugin API, including **autocompleted parameter** type and **password parameter** type. -- Explore: - - Workspaces are now **selectable at runtime**. - - Enhanced **editing capabilities** in the EasyNav editor. -- Automate: - - New **`admin user` command group** for managing user accounts in the Keycloak CMEM realm. +- Build: + - Support for **global variables** in dataset and task parameters. + - Extensions to the Python Plugin API, including **autocompleted parameter** type and **password parameter** type. +- Explore: + - Workspaces are now **selectable at runtime**. + - Enhanced **editing capabilities** in the EasyNav editor. +- Automate: + - New **`admin user` command group** for managing user accounts in the Keycloak CMEM realm. !!! warning @@ -27,11 +27,11 @@ The highlights of this release are: This release delivers the following component versions: -- eccenca DataPlatform v23.1.3 -- eccenca DataIntegration v23.1.2 -- eccenca DataIntegration Python Plugins v3.0.0 -- eccenca DataManager v23.1.5 -- eccenca Corporate Memory Control (cmemc) v23.1.3 +- eccenca DataPlatform v23.1.3 +- eccenca DataIntegration v23.1.2 +- eccenca DataIntegration Python Plugins v3.0.0 +- eccenca DataManager v23.1.5 +- eccenca Corporate Memory Control (cmemc) v23.1.3 More detailed release notes for these versions are listed below. @@ -41,97 +41,97 @@ We're excited to bring you the latest update to DataIntegration v23.1, featuring v23.1.2 of eccenca DataIntegration ships following fixes: -- Saving a transform or linking rule with an operator that references a project resource fails. -- Cannot read large Excel files from S3. +- Saving a transform or linking rule with an operator that references a project resource fails. +- Cannot read large Excel files from S3. v23.1.1 of eccenca DataIntegration ships following fixes: -- Fixed various vulnerabilities by upgrading affected libraries. -- Workflows using the "SPARQL Update query" operator fail with "Need non-empty resource manager" errors. -- use cmem-plugin-base 3.1.0 instead of RC1 -- remove some unused base image packages +- Fixed various vulnerabilities by upgrading affected libraries. +- Workflows using the "SPARQL Update query" operator fail with "Need non-empty resource manager" errors. +- use cmem-plugin-base 3.1.0 instead of RC1 +- remove some unused base image packages v23.1 of eccenca DataIntegration adds the following new features: -- Support for global variables: - - Dataset and task parameters can be set to Jinja templates. - - Templates may access configured global variables. User-defined variables will be added later. - - Global variable resolution is supported by the 'Evaluate template' transform operator. - - Disabled by default. -- Extensions to the Python Plugin API: - - Autocompleted parameter types may declare dependent parameters. - - Password plugin parameter type. - - Custom parameter types can be registered - - For details, see changelog of the cmem-plugin-base module. -- REST endpoint to search for properties in the global vocabulary cache: - - GET /api/workspace/vocabularies/property/search - - Warn of invisible characters in input fields and offer action to remove them from the input string. -- Autocompletion of graph parameters. -- Auto-completion support to linking rule 'link type' parameter. -- Improve handling of replaceable datasets: - - Datasets that can be replaced/configured in a workflow at API request time can be set in the workflow editor. - - This allows for the execution of workflows with mock data, which has not been possible with 'Variable dataset' tasks. -- Allow to config datasets as read-only to prevent accidentally writing into them. -- New resource endpoints to replace the deprecated resource endpoints. See deprecation section for more details. -- Allow to force start activity. -- Rewritten linking evaluation view. +- Support for global variables: + - Dataset and task parameters can be set to Jinja templates. + - Templates may access configured global variables. User-defined variables will be added later. + - Global variable resolution is supported by the 'Evaluate template' transform operator. + - Disabled by default. +- Extensions to the Python Plugin API: + - Autocompleted parameter types may declare dependent parameters. + - Password plugin parameter type. + - Custom parameter types can be registered + - For details, see changelog of the cmem-plugin-base module. +- REST endpoint to search for properties in the global vocabulary cache: + - GET /api/workspace/vocabularies/property/search + - Warn of invisible characters in input fields and offer action to remove them from the input string. +- Autocompletion of graph parameters. +- Auto-completion support to linking rule 'link type' parameter. +- Improve handling of replaceable datasets: + - Datasets that can be replaced/configured in a workflow at API request time can be set in the workflow editor. + - This allows for the execution of workflows with mock data, which has not been possible with 'Variable dataset' tasks. +- Allow to config datasets as read-only to prevent accidentally writing into them. +- New resource endpoints to replace the deprecated resource endpoints. See deprecation section for more details. +- Allow to force start activity. +- Rewritten linking evaluation view. v23.1 of eccenca DataIntegration introduces the following changes: -- Check token expiration (> 5s left) before sending a request to prevent unnecessary request retries. -- 'Concatenate' and 'Concatenate multiple values' transformer: - - In 'glue' parameter value support `\t`, `\n` and `\\` as escaped characters. -- Indexing of levenshtein comparisons can be configured now. -- Rename 'Constant' comparison operator to 'Constant similarity value'. -- Neo4j improvements: - - Support for paths when reading entities (forward and backward operators). - - Using a relation at the end of a path will return the URI of the node. - - The `#id` special path will return the internal node id. -- CSV dataset auto-configuration now supports detecting more encodings for the Charset parameter. -- Changed search behavior in most places to search after typing stops instead of needing to hit the ENTER key: - - In the 'Create new item' dialog hitting the Enter key now has the same effect as clicking the 'Add' button. -- Show value type label primarily instead of ID. -- Show default URI pattern example in a object rule mapping form when the source path is non-empty. -- Response body of a failed REST operator request is also added to the workflow report in addition to being logged. -- Linking execution report has a warning message when the link limit was reduced because of the config of `linking.execution.linkLimit.max`. -- Disable streaming in 'Parse JSON' operator, so backward paths can be used against it. -- Improved online documentation of many rule operators: - - Distance measures: Added information if a measure is either boolean, normalized or unbounded. - - Distance measures: Clarified what happens with multiple values for single value measures. - - Transformers, Distance measures and Aggregators: Added examples +- Check token expiration (> 5s left) before sending a request to prevent unnecessary request retries. +- 'Concatenate' and 'Concatenate multiple values' transformer: + - In 'glue' parameter value support `\t`, `\n` and `\\` as escaped characters. +- Indexing of levenshtein comparisons can be configured now. +- Rename 'Constant' comparison operator to 'Constant similarity value'. +- Neo4j improvements: + - Support for paths when reading entities (forward and backward operators). + - Using a relation at the end of a path will return the URI of the node. + - The `#id` special path will return the internal node id. +- CSV dataset auto-configuration now supports detecting more encodings for the Charset parameter. +- Changed search behavior in most places to search after typing stops instead of needing to hit the ENTER key: + - In the 'Create new item' dialog hitting the Enter key now has the same effect as clicking the 'Add' button. +- Show value type label primarily instead of ID. +- Show default URI pattern example in a object rule mapping form when the source path is non-empty. +- Response body of a failed REST operator request is also added to the workflow report in addition to being logged. +- Linking execution report has a warning message when the link limit was reduced because of the config of `linking.execution.linkLimit.max`. +- Disable streaming in 'Parse JSON' operator, so backward paths can be used against it. +- Improved online documentation of many rule operators: + - Distance measures: Added information if a measure is either boolean, normalized or unbounded. + - Distance measures: Clarified what happens with multiple values for single value measures. + - Transformers, Distance measures and Aggregators: Added examples v23.1 of eccenca DataIntegration ships following fixes: -- Layout breaks on small screens on detail pages of the workspace. -- Mapping suggestion list is empty when there is no matching response even though source paths exist. -- Active Learning shows incorrect entity values. -- Add notes dialog keeps focus when workflow is executed and running. -- Race condition in project/task tag selection. -- Dataset auto-configure parameter changes not set for parameters that support auto-completion. -- Label and description of existing root/object rules cannot be changed. -- DI writes invalid XML, if the last segment of a URI starts with a number. -- Optimize peak endpoint if only one path is requested. -- Python Plugin Environment: package dependencies can not update the base requirements anymore. -- Spinner is being shown eternally when no comparison pairs have been found in the link learning. -- Value path auto-completion can suggest wrong paths if backward paths exist in the paths cache. -- Show spinner while transform examples are requested from the backend. -- Abort a not fully consumed S3 input stream instead of closing it which leads to warnings. -- Date parser fails when no input/output pattern is selected even though an alternative input/output pattern is given. -- Dependent parameter auto-completion using default values of other parameters. -- Support replaceable/variable datasets in nested workflows. -- Display info message when a parameter is disabled because it depends on other parameters to be set. -- 'Fix URI' operator trims the URI before fixing it and tries better to maintain the original URI with only the invalid characters encoded. -- Task completion message is shown without executing the transformation. -- Evaluation in mapping rule editor does not work when inside object mappings. -- Show error message when project import fails because of errors detected in the backend instead of closing the project import modal. -- Linking editor evaluation toolbar component issues. -- Levensthein indexing slow if combined conjunctively. -- Transform execution tab layout issues. +- Layout breaks on small screens on detail pages of the workspace. +- Mapping suggestion list is empty when there is no matching response even though source paths exist. +- Active Learning shows incorrect entity values. +- Add notes dialog keeps focus when workflow is executed and running. +- Race condition in project/task tag selection. +- Dataset auto-configure parameter changes not set for parameters that support auto-completion. +- Label and description of existing root/object rules cannot be changed. +- DI writes invalid XML, if the last segment of a URI starts with a number. +- Optimize peak endpoint if only one path is requested. +- Python Plugin Environment: package dependencies can not update the base requirements anymore. +- Spinner is being shown eternally when no comparison pairs have been found in the link learning. +- Value path auto-completion can suggest wrong paths if backward paths exist in the paths cache. +- Show spinner while transform examples are requested from the backend. +- Abort a not fully consumed S3 input stream instead of closing it which leads to warnings. +- Date parser fails when no input/output pattern is selected even though an alternative input/output pattern is given. +- Dependent parameter auto-completion using default values of other parameters. +- Support replaceable/variable datasets in nested workflows. +- Display info message when a parameter is disabled because it depends on other parameters to be set. +- 'Fix URI' operator trims the URI before fixing it and tries better to maintain the original URI with only the invalid characters encoded. +- Task completion message is shown without executing the transformation. +- Evaluation in mapping rule editor does not work when inside object mappings. +- Show error message when project import fails because of errors detected in the backend instead of closing the project import modal. +- Linking editor evaluation toolbar component issues. +- Levensthein indexing slow if combined conjunctively. +- Transform execution tab layout issues. v23.1 of eccenca DataIntegration introduced the following deprecations: -- Resource endpoints: - - All resources endpoints that have the _file path_ (`workspace/projects/:project/resources/:name`) encoded in the URL path are now deprecated. The files endpoints using a _query parameter_ for the path should be used now. +- Resource endpoints: + - All resources endpoints that have the _file path_ (`workspace/projects/:project/resources/:name`) encoded in the URL path are now deprecated. The files endpoints using a _query parameter_ for the path should be used now. ## eccenca DataIntegration Python Plugins v3.0.0 @@ -139,7 +139,7 @@ Corporate Memory v23.1 includes the DataIntegration Python Plugins support in ve v3.0.0 of eccenca DataIntegration Python Plugins adds the following new features: -- Autocompleted parameter types may declare dependent parameters. For instance, a parameter `city` may declare that its completed values depend on another parameter 'country': +- Autocompleted parameter types may declare dependent parameters. For instance, a parameter `city` may declare that its completed values depend on another parameter 'country': ```py class CityParameterType(StringParameterType): @@ -152,7 +152,7 @@ v3.0.0 of eccenca DataIntegration Python Plugins adds the following new features return ... ``` -- Password plugin parameter type. Passwords will be encrypted in the backend and not shown to users: +- Password plugin parameter type. Passwords will be encrypted in the backend and not shown to users: ```py @Plugin(label="My Plugin") @@ -164,7 +164,7 @@ v3.0.0 of eccenca DataIntegration Python Plugins adds the following new features self.password.decrypt() ``` -- Custom parameter types can be registered. See implementation of `PasswordParameterType` for an example. +- Custom parameter types can be registered. See implementation of `PasswordParameterType` for an example. ## eccenca DataManager v23.1.5 @@ -172,41 +172,41 @@ We are excited to announce the latest update to DataManager v23.1, which introdu v23.1.5 of eccenca DataManager ships following fixes: -- Fixed download of query result in query editor. -- Setting the defaultGraph of a explore workspace configuration no longer prevents the Navigation box from loading. -- Fixes in the LinkRules modules: Result-Details, Rule-Deletions, Property-Search +- Fixed download of query result in query editor. +- Setting the defaultGraph of a explore workspace configuration no longer prevents the Navigation box from loading. +- Fixes in the LinkRules modules: Result-Details, Rule-Deletions, Property-Search v23.1.4 of eccenca DataManager ships following changes: -- Switch from iframe to redirect based login view. - - Known issues: Interactions after the timeout do not always trigger a reload and simply shows error messages or empty results. Using the navigation bar triggers a reload. +- Switch from iframe to redirect based login view. + - Known issues: Interactions after the timeout do not always trigger a reload and simply shows error messages or empty results. Using the navigation bar triggers a reload. v23.1.3 of eccenca DataManager ships following fixes: -- use latest debian:bullseye-20230411-slim base image -- use wget instead of curl +- use latest debian:bullseye-20230411-slim base image +- use wget instead of curl v23.1.2 of eccenca DataManager was a redacted build due to incomplete merge. v23.1.1 of eccenca DataManager ships following fixes: -- Fixes link rules creation dialogue setting a target property. +- Fixes link rules creation dialogue setting a target property. v23.1 of eccenca DataManager adds the following new features: -- Workspaces are selectable at runtime. -- Routes can include a workspace selection. -- Added Editing capabilities to the EasyNav editor. +- Workspaces are selectable at runtime. +- Routes can include a workspace selection. +- Added Editing capabilities to the EasyNav editor. v23.1 of eccenca DataManager introduces the following changes: -- Configuration is now fully retrieved from DataPlatform, the included Spring Boot based backend is solely delivering the javascript frontend. -- The configuration can be changed at runtime using a frontend in the `/admin` Module. Changes are visible with the next full browser reload. -- Authentication is now based on the OAuth2 Code Flow. +- Configuration is now fully retrieved from DataPlatform, the included Spring Boot based backend is solely delivering the javascript frontend. +- The configuration can be changed at runtime using a frontend in the `/admin` Module. Changes are visible with the next full browser reload. +- Authentication is now based on the OAuth2 Code Flow. v23.1 of eccenca DataManager ships following fixes: -- Removed session token from URL. +- Removed session token from URL. ## eccenca DataPlatform v23.1.3 @@ -214,50 +214,50 @@ We're excited to announce the latest update to DataPlatform v23.1, featuring sig v23.1.3 of eccenca DataPlatform ships following fixes: -- Fix wrong calculation of write graph access under certain conditions. +- Fix wrong calculation of write graph access under certain conditions. v23.1.2 of eccenca DataPlatform ships following changes: -- DP/Infinispan: session timeout increased to 10h -- Login: switch from iframe to redirect based flow +- DP/Infinispan: session timeout increased to 10h +- Login: switch from iframe to redirect based flow v23.1.1 of eccenca DataPlatform ships following fixes: -- docker image: use latest debian:bullseye-20230411-slim base image -- docker image: wget instead of curl +- docker image: use latest debian:bullseye-20230411-slim base image +- docker image: wget instead of curl v23.1 of eccenca DataPlatform adds the following new features: -- Added ability to use dynamic access conditions -- Added graph for infos about logged in users (iri, login): - - Can be (de)activated using property `authorization.userInfoGraph.active` (default: true) -- Workspace Selection and Configuration: - - Activate OAuth 2.0 client role permanently - - Redirect login page to (exactly) one configured resource provider - - REST endpoints for workspace configuration +- Added ability to use dynamic access conditions +- Added graph for infos about logged in users (iri, login): + - Can be (de)activated using property `authorization.userInfoGraph.active` (default: true) +- Workspace Selection and Configuration: + - Activate OAuth 2.0 client role permanently + - Redirect login page to (exactly) one configured resource provider + - REST endpoints for workspace configuration v23.1 of eccenca DataPlatform introduces the following changes: -- Integrate infinispan as sole cache provider: - - Enables clustering of DataPlatform instances - - clustering can be activated by `spring.cache.infinispan.mode=CLUSTER` - - Removed property `files.maintenanceCron` (housekeeping done by infinispan) - - Added property `files.storageDirectory` for configuring shared directory between multiple DataPlatform instances - - Replaced property `proxy.cacheInvalidationCron` with `proxy.cacheExpiration` (no scheduled flush anymore but cache expiration as default) -- Changed logic of resolving user rights through access conditions - performance optimized +- Integrate infinispan as sole cache provider: + - Enables clustering of DataPlatform instances + - clustering can be activated by `spring.cache.infinispan.mode=CLUSTER` + - Removed property `files.maintenanceCron` (housekeeping done by infinispan) + - Added property `files.storageDirectory` for configuring shared directory between multiple DataPlatform instances + - Replaced property `proxy.cacheInvalidationCron` with `proxy.cacheExpiration` (no scheduled flush anymore but cache expiration as default) +- Changed logic of resolving user rights through access conditions - performance optimized v23.1 of eccenca DataPlatform ships following fixes: -- Prevent injection of formulas in Excel/CSV exports -- Diagnostic store operations / query rewrite log on logging topic `com.eccenca.elds.backend.sparql.query.diagnostic` - must be set to TRACE: - - Activated update result statistics in existing query result logger -- Missing access condition action resource for EasyNav added +- Prevent injection of formulas in Excel/CSV exports +- Diagnostic store operations / query rewrite log on logging topic `com.eccenca.elds.backend.sparql.query.diagnostic` - must be set to TRACE: + - Activated update result statistics in existing query result logger +- Missing access condition action resource for EasyNav added v23.1 of eccenca DataPlatform removed the following features and configurations: -- Deprecated properties under authorization.accessConditions - - `authorization.accessConditions.graph`: used graph is always the default graph from bootstrap - - `authorization.accessConditions.url`: url as source for access condition not supported anymore +- Deprecated properties under authorization.accessConditions + - `authorization.accessConditions.graph`: used graph is always the default graph from bootstrap + - `authorization.accessConditions.url`: url as source for access condition not supported anymore ## eccenca Corporate Memory Control (cmemc) v23.1.3 @@ -271,34 +271,34 @@ v23.1.3 of eccenca Corporate Memory Control introduces the following security up v23.1.2 of eccenca Corporate Memory Control introduces the following fixes: -- broken installation due to `urllib3` dependency +- broken installation due to `urllib3` dependency - `urllib3>=2` was released 2023-04-26 but is broken with this error: `ImportError: cannot import name 'appengine' from 'urllib3.contrib'` - cmemc requested any version and not `^1.26.15` of this library, which resulted in broken installations with pip beginning from 2023-04-26 - quick fix to solve this without updating cmemc: `pip install urllib3==1.26.15` in the cmemc virtual env v23.1.1 of eccenca Corporate Memory Control introduces the following changes: -- remove some unneeded packages from docker image -- switch to python 3.11.3 base image and tests +- remove some unneeded packages from docker image +- switch to python 3.11.3 base image and tests v23.1 of eccenca Corporate Memory Control adds the following new features: -- `admin status` command: - - option `--exit-1` to specify, when to return non-zero exit code - - currently set to `never`, this will be changed to `always` in the future -- `admin user` command group: - - `create` command - add a user account to the keycloak CMEM realm - - `delete` command - remove a user account from the keycloak CMEM realm - - `list` command - list user accounts in the keycloak CMEM realm - - `password` command - change the accounts password - - `update` command - change a user account in the keycloak CMEM realm -- optional `KEYCLOAK_BASE_URI` config environment -- optional `KEYCLOAK_REALM_ID` config environment +- `admin status` command: + - option `--exit-1` to specify, when to return non-zero exit code + - currently set to `never`, this will be changed to `always` in the future +- `admin user` command group: + - `create` command - add a user account to the keycloak CMEM realm + - `delete` command - remove a user account from the keycloak CMEM realm + - `list` command - list user accounts in the keycloak CMEM realm + - `password` command - change the accounts password + - `update` command - change a user account in the keycloak CMEM realm +- optional `KEYCLOAK_BASE_URI` config environment +- optional `KEYCLOAK_REALM_ID` config environment v23.1 of eccenca Corporate Memory Control introduced the following deprecations: -- `admin status` command `--exit-1` option default - - currently set to `never`, this will be changed to `always` in a future release +- `admin status` command `--exit-1` option default + - currently set to `never`, this will be changed to `always` in a future release ## Migration Notes @@ -312,9 +312,9 @@ v23.1 of eccenca Corporate Memory Control introduced the following deprecations: ### eccenca DataIntegration -- Resource endpoints: - - All resources endpoints that have the _file path_ (`workspace/projects/:project/resources/:name`) encoded in the URL path are now deprecated. - - Use corresponding endpoints starting with `workspace/projects/:project/files` instead, using a _query parameter_ for the file path. +- Resource endpoints: + - All resources endpoints that have the _file path_ (`workspace/projects/:project/resources/:name`) encoded in the URL path are now deprecated. + - Use corresponding endpoints starting with `workspace/projects/:project/files` instead, using a _query parameter_ for the file path. ### eccenca DataIntegration Python Plugins @@ -335,10 +335,10 @@ def label(self, value: str, depend_on_parameter_values: list[Any], context: Plug ### eccenca DataManager -- A manual migration for the graph based configuration of the EasyNav configuration and the graph list configuration of the explore module is necessary. -- A manual migration for the `.yml` based DataManager configuration is necessary. -- The new web based configuration tool can be used to migrate, create and manage your DataManager (workspace) configuration +- A manual migration for the graph based configuration of the EasyNav configuration and the graph list configuration of the explore module is necessary. +- A manual migration for the `.yml` based DataManager configuration is necessary. +- The new web based configuration tool can be used to migrate, create and manage your DataManager (workspace) configuration ### eccenca DataPlatform -- Deprecated properties under `authorization.accessConditions` have been removed. The used graph is always the default graph from bootstrap, and URL as a source for access conditions is not supported anymore. +- Deprecated properties under `authorization.accessConditions` have been removed. The used graph is always the default graph from bootstrap, and URL as a source for access conditions is not supported anymore. diff --git a/docs/release-notes/corporate-memory-23-2/index.md b/docs/release-notes/corporate-memory-23-2/index.md index 9d3e49bc..7da43106 100644 --- a/docs/release-notes/corporate-memory-23-2/index.md +++ b/docs/release-notes/corporate-memory-23-2/index.md @@ -12,25 +12,25 @@ Corporate Memory 23.2.1 is the first patch release in the 23.2 release line. The highlights of this release are: -- Build: - - Support for user managed **project variables** in dataset and task parameters. - - All new UIs for **transformation evaluation and reference links**. -- Explore: - - New feature in Easynav like - - **nodes context menu**, - - **long label support**, - - advanced **graph selection dialog** and - - **automatic node layout**. -- Automate: - - New **`admin client` command group** for managing client accounts in the Keycloak CMEM realm. +- Build: + - Support for user managed **project variables** in dataset and task parameters. + - All new UIs for **transformation evaluation and reference links**. +- Explore: + - New feature in Easynav like + - **nodes context menu**, + - **long label support**, + - advanced **graph selection dialog** and + - **automatic node layout**. +- Automate: + - New **`admin client` command group** for managing client accounts in the Keycloak CMEM realm. This release delivers the following component versions: -- eccenca DataPlatform v23.2.1 -- eccenca DataIntegration v23.2.1 -- eccenca DataIntegration Python Plugins v4.1.0 -- eccenca DataManager v23.2 -- eccenca Corporate Memory Control (cmemc) v23.2 +- eccenca DataPlatform v23.2.1 +- eccenca DataIntegration v23.2.1 +- eccenca DataIntegration Python Plugins v4.1.0 +- eccenca DataManager v23.2 +- eccenca Corporate Memory Control (cmemc) v23.2 More detailed release notes for these versions are listed below. @@ -40,70 +40,70 @@ We're excited to bring you the latest update to DataIntegration v23.2, featuring v23.2.1 of eccenca DataIntegration ships the following improvements and fixes: -- Added overview listing all available keyboard shortcuts, available by `?` key, or from the user menu. -- Improved vocabulary (needs to be installed separately). -- The file API will set the content type based on the file extension: - - For instance, for a file ending in `.json` the `Content-Type` header will be set to `application/json`. -- Upgraded several libraries to fix vulnerabilities. +- Added overview listing all available keyboard shortcuts, available by `?` key, or from the user menu. +- Improved vocabulary (needs to be installed separately). +- The file API will set the content type based on the file extension: + - For instance, for a file ending in `.json` the `Content-Type` header will be set to `application/json`. +- Upgraded several libraries to fix vulnerabilities. v23.2 of eccenca DataIntegration adds the following new features: -- User-defined project variables: - - Can be used in dataset and task parameters and in the template transform operator. - - Variables may use templates that access other preceding project variables or globally configured variables. -- All new transform evaluation UI. -- All new reference links view. -- Extensions to transform rule and linking rule editors: - - Support for setting a language filter for a path operator conveniently without having to use the language filter syntax. - - Partial linking and transform rule (tree) evaluation. -- Support fixing tasks that have failed loading: - - Allow a user to reload a task. The user may change the original parameters of the task. - - `POST /api/workspace/projects/{projectId}/reloadFailedTask` endpoint that reloads a task with optionally updated parameter values. - - `GET /api/workspace/projects/{projectId}/failedTaskParameters/{taskId}` endpoint that fetches the original parameter values of a failed task. -- Added "Concatenate pairwise" transform operator. -- Make transform suggestion matching link spec / workflow customizable. -- API extensions: - - Transform evaluation endpoint: `/transform/tasks/{projectId}/{transformTaskId}/rule/{ruleId}/evaluated` - - Added endpoints for uploading and downloading files of datasets. - - REST endpoint to fetch dataset characteristics. - - `GET /api/workspace/projects/(projectId)/datasets/{datasetId}/characteristics` - - API endpoint to fetch IDs of file based datasets. - - `/api/core/datasets/resourceBased` - - REST endpoint to copy an arbitrary linking task to the matching linking task that will be used in the mapping suggestion. - - `POST /ontologyMatching/replaceOntologyMatchingLinkSpec` - - Both `matching.external.projectId` and `matching.external.linkSpecId` must be configured - - REST endpoint that generates an ontology matching project and linking tasks based on a specific transformation task. - - `POST /ontologyMatching/generateMatchingLinkRule` +- User-defined project variables: + - Can be used in dataset and task parameters and in the template transform operator. + - Variables may use templates that access other preceding project variables or globally configured variables. +- All new transform evaluation UI. +- All new reference links view. +- Extensions to transform rule and linking rule editors: + - Support for setting a language filter for a path operator conveniently without having to use the language filter syntax. + - Partial linking and transform rule (tree) evaluation. +- Support fixing tasks that have failed loading: + - Allow a user to reload a task. The user may change the original parameters of the task. + - `POST /api/workspace/projects/{projectId}/reloadFailedTask` endpoint that reloads a task with optionally updated parameter values. + - `GET /api/workspace/projects/{projectId}/failedTaskParameters/{taskId}` endpoint that fetches the original parameter values of a failed task. +- Added "Concatenate pairwise" transform operator. +- Make transform suggestion matching link spec / workflow customizable. +- API extensions: + - Transform evaluation endpoint: `/transform/tasks/{projectId}/{transformTaskId}/rule/{ruleId}/evaluated` + - Added endpoints for uploading and downloading files of datasets. + - REST endpoint to fetch dataset characteristics. + - `GET /api/workspace/projects/(projectId)/datasets/{datasetId}/characteristics` + - API endpoint to fetch IDs of file based datasets. + - `/api/core/datasets/resourceBased` + - REST endpoint to copy an arbitrary linking task to the matching linking task that will be used in the mapping suggestion. + - `POST /ontologyMatching/replaceOntologyMatchingLinkSpec` + - Both `matching.external.projectId` and `matching.external.linkSpecId` must be configured + - REST endpoint that generates an ontology matching project and linking tasks based on a specific transformation task. + - `POST /ontologyMatching/generateMatchingLinkRule` v23.2 of eccenca DataIntegration introduces the following changes: -- Close user menu automatically. -- Linking rule config: - - Add 'Inverse link type' parameter that defined a URI that is generated from the target to the source resource, i.e. the inverse of the 'link type' parameter. - - Add 'Is reflexive' parameter that when enabled does not link resources with themselves. -- In a workflow datasets with schema-less inputs, e.g. workflows, other datasets, are not considered to be outputs datasets anymore. -- Variable workflow API: - - Support uploading large input files via multipart/form-data request - - Support custom mime type "application/x-plugin-" in CONTENT-TYPE or ACCEPT in order to support all file based dataset plugins. - - Support query parameters 'config-dataSourceConfig-' and 'config-dataSinkConfig-' to configure dataset parameters of the data source and sink. -- Added read-only / uriProperty dataset attributes to endpoint responses. -- forward cmem-plugin-base to v4.1.0, via base image v2.2.0 -- The handling of errors in transform rules has been aligned with what is already shown in the evaluation: - - If a nested operator throws a validation error (e.g., if the input value is not a number for numeric operators), this no longer leads to a failure of the entire rule. - - The error will be added to the execution report. - - Failed operators will return no value. +- Close user menu automatically. +- Linking rule config: + - Add 'Inverse link type' parameter that defined a URI that is generated from the target to the source resource, i.e. the inverse of the 'link type' parameter. + - Add 'Is reflexive' parameter that when enabled does not link resources with themselves. +- In a workflow datasets with schema-less inputs, e.g. workflows, other datasets, are not considered to be outputs datasets anymore. +- Variable workflow API: + - Support uploading large input files via multipart/form-data request + - Support custom mime type "application/x-plugin-" in CONTENT-TYPE or ACCEPT in order to support all file based dataset plugins. + - Support query parameters 'config-dataSourceConfig-' and 'config-dataSinkConfig-' to configure dataset parameters of the data source and sink. +- Added read-only / uriProperty dataset attributes to endpoint responses. +- forward cmem-plugin-base to v4.1.0, via base image v2.2.0 +- The handling of errors in transform rules has been aligned with what is already shown in the evaluation: + - If a nested operator throws a validation error (e.g., if the input value is not a number for numeric operators), this no longer leads to a failure of the entire rule. + - The error will be added to the execution report. + - Failed operators will return no value. v23.2 of eccenca DataIntegration ships the following fixes: -- Excel plugins are not available (CMEM-5088). -- Transform/linking tasks with operators that use projects resources, e.g. Excel transform, cannot be copied/cloned to other projects (CMEM-5065). -- Using a target property ending in `/valueOf` for a value mapping rule breaks the mapping editor (CMEM-5059). -- Re-configured pure input dataset in a workflow should not be seen as an output dataset (CMEM-5058). -- Prioritized and blocking activities should be run in a fork join pool as well (CMEM-4856). -- Workflow(s) info endpoints return error (500) when a workflow is invalid (CMEM-5099): - - There is a 'warnings' property in the returned JSON that describes which information is missing and why. -- Remove broken legacy navigation menu from client error template that is shown e.g. when a user is not authorized to use DI (CMEM-4977). -- Mapping editor: the path suggestion that exactly matches the search query is not shown in the list of suggestions (CMEM-5084). +- Excel plugins are not available (CMEM-5088). +- Transform/linking tasks with operators that use projects resources, e.g. Excel transform, cannot be copied/cloned to other projects (CMEM-5065). +- Using a target property ending in `/valueOf` for a value mapping rule breaks the mapping editor (CMEM-5059). +- Re-configured pure input dataset in a workflow should not be seen as an output dataset (CMEM-5058). +- Prioritized and blocking activities should be run in a fork join pool as well (CMEM-4856). +- Workflow(s) info endpoints return error (500) when a workflow is invalid (CMEM-5099): + - There is a 'warnings' property in the returned JSON that describes which information is missing and why. +- Remove broken legacy navigation menu from client error template that is shown e.g. when a user is not authorized to use DI (CMEM-4977). +- Mapping editor: the path suggestion that exactly matches the search query is not shown in the list of suggestions (CMEM-5084). ## eccenca DataIntegration Python Plugins v4.1.0 @@ -111,10 +111,10 @@ Corporate Memory v23.2 includes the DataIntegration Python Plugins support in ve v4.1.0 of eccenca DataIntegration Python Plugins adds the following new features: -- use `post_resource` api in `write_to_dataset` function to update dataset file resource -- use cmempy 23.2 -- upgrade dependencies -- enforce usage of Python 3.11 +- use `post_resource` api in `write_to_dataset` function to update dataset file resource +- use cmempy 23.2 +- upgrade dependencies +- enforce usage of Python 3.11 ## eccenca DataManager v23.2 @@ -122,46 +122,46 @@ We are excited to announce the latest update to DataManager v23.2, which introdu v23.2 of eccenca DataManager adds the following new features: -- Added advanced options ("Inverse Linking Property" and "Irreflexive Linking") in a Link rule setup Dialog. -- Automatic layouts in Easynav. -- Configurable defaultTimeout for queries. +- Added advanced options ("Inverse Linking Property" and "Irreflexive Linking") in a Link rule setup Dialog. +- Automatic layouts in Easynav. +- Configurable defaultTimeout for queries. v23.2 of eccenca DataManager ships the following changes: -- Auto-close user menu. -- Delete Resource Dialog was overhauled, and can now trigger a `shui:onDeleteUpdate` query. -- Adjusted the payload when saving a custom or the default workspace config so that it has only the modified values. -- Better error message for empty string after stopwords remove in the Easynav module. -- Allow stale facets initially to improve loading timing. After stale facets are fetched, actual ones will be requested in background. -- Easynav: - - Long labels are shown in two lines. - - Save option for saving only the ontology changes. - - Context menu on resource nodes is re-enables. - - Query based entry into Easynav via the "graph" tab in explore. - - Context graph name is shown in breadcrumbs. - - Detailed filtering in the visualization catalog. - - Search bar centers on the node. - - All possible connections are shown for a node, not just the ones with data. - - Rich graph selection widget. -- Pathbuilder: - - Added subpaths option if `hierarchyEnable` is true. - - Added server search for subject and predicate. - - Added support for longer labels in resource selector. - - Added helper texts for subject selection and subpaths selection. - - Added depictions for subjects and predicates. - - Added different colors for subpaths. -- Explore/Properties View: - - Geo-coordinates are only shown, when a map server is configured. - - The wording of the error message when no selected graph found is changed. +- Auto-close user menu. +- Delete Resource Dialog was overhauled, and can now trigger a `shui:onDeleteUpdate` query. +- Adjusted the payload when saving a custom or the default workspace config so that it has only the modified values. +- Better error message for empty string after stopwords remove in the Easynav module. +- Allow stale facets initially to improve loading timing. After stale facets are fetched, actual ones will be requested in background. +- Easynav: + - Long labels are shown in two lines. + - Save option for saving only the ontology changes. + - Context menu on resource nodes is re-enables. + - Query based entry into Easynav via the "graph" tab in explore. + - Context graph name is shown in breadcrumbs. + - Detailed filtering in the visualization catalog. + - Search bar centers on the node. + - All possible connections are shown for a node, not just the ones with data. + - Rich graph selection widget. +- Pathbuilder: + - Added subpaths option if `hierarchyEnable` is true. + - Added server search for subject and predicate. + - Added support for longer labels in resource selector. + - Added helper texts for subject selection and subpaths selection. + - Added depictions for subjects and predicates. + - Added different colors for subpaths. +- Explore/Properties View: + - Geo-coordinates are only shown, when a map server is configured. + - The wording of the error message when no selected graph found is changed. v23.2 of eccenca DataIntegration ships the following fixes: -- Replaced an obsolete `LabelResolutionApi` by RTK-Query to catch 401 errors (lost session/authorization) and bring cache (CMEM-4979). -- Restored notifications toast (CMEM-4979). -- Enabled deletion of large tables (CMEM-4898). -- Added `mapServer` options in explore config (CMEM-4926). -- Error message shows when trying to upload broken file to the graph (CMEM-4704). -- Fixed error with lost graph list tabs on workspace switch (CMEM-5087). +- Replaced an obsolete `LabelResolutionApi` by RTK-Query to catch 401 errors (lost session/authorization) and bring cache (CMEM-4979). +- Restored notifications toast (CMEM-4979). +- Enabled deletion of large tables (CMEM-4898). +- Added `mapServer` options in explore config (CMEM-4926). +- Error message shows when trying to upload broken file to the graph (CMEM-4704). +- Fixed error with lost graph list tabs on workspace switch (CMEM-5087). ## eccenca DataPlatform v23.2.1 @@ -169,61 +169,61 @@ We're excited to bring you the latest update to DataPlatform v23.2, featuring nu v23.2.1 of eccenca DataPlatform ships following fixes: -- Bootstrap Data: Removed obsolete DataIntegration vocabulary from shape catalog +- Bootstrap Data: Removed obsolete DataIntegration vocabulary from shape catalog v23.2 of eccenca DataPlatform adds the following new features: -- Bootstrap Data: allow `sh:order` for `sh:NodesShape`. -- Dynamic access conditions backend functionality. -- Add optional facets request param for getting (possibly) stale cached values. -- Provenance metadata for Easynav visualizations. -- Added property `hierarchyEnable` to Link Rule Modul. -- Added properties for `mapServer` to Explore Modul. -- Add node shape property shape for `onDeleteUpdate` of resource. -- Added DI vocabulary. -- Added provenance metadata to visualization catalogue entries. +- Bootstrap Data: allow `sh:order` for `sh:NodesShape`. +- Dynamic access conditions backend functionality. +- Add optional facets request param for getting (possibly) stale cached values. +- Provenance metadata for Easynav visualizations. +- Added property `hierarchyEnable` to Link Rule Modul. +- Added properties for `mapServer` to Explore Modul. +- Add node shape property shape for `onDeleteUpdate` of resource. +- Added DI vocabulary. +- Added provenance metadata to visualization catalogue entries. v23.2 of eccenca DataPlatform ships the following changes: -- PropertyUsage endpoint delivers language tags. -- Make order of node shapes editable. -- Resolving of depictions for resource along node shape order. -- Viewing labels of NodeShapes as `SHACL.Name` - changed from `RDFS.Label`. -- Resource deletion applies Symmetric Concise Bound Description including incoming links. -- Show full list of access conditions for users with access condition management action. -- Add configuration option to switch between ontodia / graph tab in explore view. +- PropertyUsage endpoint delivers language tags. +- Make order of node shapes editable. +- Resolving of depictions for resource along node shape order. +- Viewing labels of NodeShapes as `SHACL.Name` - changed from `RDFS.Label`. +- Resource deletion applies Symmetric Concise Bound Description including incoming links. +- Show full list of access conditions for users with access condition management action. +- Add configuration option to switch between ontodia / graph tab in explore view. v23.2 of eccenca DataPlatform ships following fixes: -- Prevent upload of incorrect file URIs (CMEM-4360). -- Reintroduce `defaultTimeout` for UI queries (CMEM-5100). -- Make property `defaultGraph` able to be overwritten in custom workspace (CMEM-4902). -- Fix labels of node shapes from `RDFS.Label` to `SHACL.Name` (CMEM-4743). +- Prevent upload of incorrect file URIs (CMEM-4360). +- Reintroduce `defaultTimeout` for UI queries (CMEM-5100). +- Make property `defaultGraph` able to be overwritten in custom workspace (CMEM-4902). +- Fix labels of node shapes from `RDFS.Label` to `SHACL.Name` (CMEM-4743). ## eccenca Corporate Memory Control (cmemc) v23.2 v23.2 of eccenca Corporate Memory Control adds the following new features: -- `admin user password` command: - - option `--request-change` added, to send a email to user to reset the password -- `dataset create` command: - - add `readOnly` and `uriProperty` keys for the `-p/--parameter` option -- `admin client` command group: - - `list` command - list client accounts - - `open` command - Open clients in the browser - - `secret` command - Get or generate a new secret for a client account -- `project create` command: - - new option `--from-transformation` to create a mapping suggestion project +- `admin user password` command: + - option `--request-change` added, to send a email to user to reset the password +- `dataset create` command: + - add `readOnly` and `uriProperty` keys for the `-p/--parameter` option +- `admin client` command group: + - `list` command - list client accounts + - `open` command - Open clients in the browser + - `secret` command - Get or generate a new secret for a client account +- `project create` command: + - new option `--from-transformation` to create a mapping suggestion project v23.2 of eccenca Corporate Memory Control introduces the following changes: -- `dataset upload` command: - - use new endpoint which is aware of read-only datasets -- `workflow io` command: - - use of extended io endpoint - - allows for uploading bigger files - - allows for more input and output mimetypes - - change default output to JSON +- `dataset upload` command: + - use new endpoint which is aware of read-only datasets +- `workflow io` command: + - use of extended io endpoint + - allows for uploading bigger files + - allows for more input and output mimetypes + - change default output to JSON ## Migration Notes diff --git a/docs/release-notes/corporate-memory-23-3/index.md b/docs/release-notes/corporate-memory-23-3/index.md index 05719914..edc852c7 100644 --- a/docs/release-notes/corporate-memory-23-3/index.md +++ b/docs/release-notes/corporate-memory-23-3/index.md @@ -12,21 +12,21 @@ Corporate Memory 23.3.2 is the second patch release in the 23.3 release line. The highlights of this release are: -- Explore and Author: - - new **[charts catalog](../../explore-and-author/charts-catalog/index.md)** module added, which allows for defining BI widgets / charts which can be integrated into shapes - - preview release of our generative AI / LLM based **Ontology and Query Assistant** -- Build: - - operate BUILD like never before by using the new **keyboard shortcuts** (press "?" in the build module to learn the details) - - several **improvements to the workflows view**: create new datasets and other workflow-operators in place, dependencies and execution order is now explicitly modeled, show schema or ports -- Automate: - - new **`project variable` command group** plus several addition to existing commands +- Explore and Author: + - new **[charts catalog](../../explore-and-author/charts-catalog/index.md)** module added, which allows for defining BI widgets / charts which can be integrated into shapes + - preview release of our generative AI / LLM based **Ontology and Query Assistant** +- Build: + - operate BUILD like never before by using the new **keyboard shortcuts** (press "?" in the build module to learn the details) + - several **improvements to the workflows view**: create new datasets and other workflow-operators in place, dependencies and execution order is now explicitly modeled, show schema or ports +- Automate: + - new **`project variable` command group** plus several addition to existing commands This release delivers the following component versions: -- eccenca DataIntegration v23.3.2 -- eccenca DataManager v23.3.1 -- eccenca DataPlatform v23.3.1 -- eccenca Corporate Memory Control (cmemc) v23.3.0 +- eccenca DataIntegration v23.3.2 +- eccenca DataManager v23.3.1 +- eccenca DataPlatform v23.3.1 +- eccenca Corporate Memory Control (cmemc) v23.3.0 More detailed release notes for these versions are listed below. @@ -36,68 +36,68 @@ We're excited to bring you the latest update to DataIntegration v23.3, featuring v23.3.2 of DataIntegration ships the following fixes: -- Entities with values larger than 65k cannot be serialized. -- JSON property path evaluation fails for missing key. +- Entities with values larger than 65k cannot be serialized. +- JSON property path evaluation fails for missing key. v23.3.1 of DataIntegration ships the following improvements: -- Workflow operator validates XML datasets against a provided XML Schema. -- Support entering a custom ID when cloning a project or project task. -- "Evaluate Template" operator has a new option for evaluating the template on the entire input set at once. +- Workflow operator validates XML datasets against a provided XML Schema. +- Support entering a custom ID when cloning a project or project task. +- "Evaluate Template" operator has a new option for evaluating the template on the entire input set at once. v23.3.1 of DataIntegration ships the following fixes: -- Long task parameter values cannot be fully seen in task config preview. -- Project export does not fail if project files cannot be read. -- Unexpected inputs of a node are not executed anymore. -- Transform report does not count entities in child mappings. -- Rule operator parameter auto-complete default values do not have a label when creating a new operator. +- Long task parameter values cannot be fully seen in task config preview. +- Project export does not fail if project files cannot be read. +- Unexpected inputs of a node are not executed anymore. +- Transform report does not count entities in child mappings. +- Rule operator parameter auto-complete default values do not have a label when creating a new operator. v23.3.0 of DataIntegration adds the following new features: -- plugin base library updated to v4.3.0 ([changelog](https://github.com/eccenca/cmem-plugin-base/blob/main/CHANGELOG.md)) -- Support for custom plugin icon. -- New `distinct by` Workflow operator that removes duplicated entities based on a user-defined path. -- Endpoint to download the DataIntegration vocabulary. -- Mappings allow custom target types - - If the new custom data type is selected, a new input field _type_ allows the user to enter a type URI. -- Added variables widget to Workflow view. -- Added hotkey access to actions in DI workspace. -- Workflow editor: - - Validate ports and connections and show warnings for found issues. - - Have menu option to show input/output schema for ports that either expect or output a fixed schema. - - Support dependency connections between workflow nodes to specify non-data execution dependencies. +- plugin base library updated to v4.3.0 ([changelog](https://github.com/eccenca/cmem-plugin-base/blob/main/CHANGELOG.md)) +- Support for custom plugin icon. +- New `distinct by` Workflow operator that removes duplicated entities based on a user-defined path. +- Endpoint to download the DataIntegration vocabulary. +- Mappings allow custom target types + - If the new custom data type is selected, a new input field _type_ allows the user to enter a type URI. +- Added variables widget to Workflow view. +- Added hotkey access to actions in DI workspace. +- Workflow editor: + - Validate ports and connections and show warnings for found issues. + - Have menu option to show input/output schema for ports that either expect or output a fixed schema. + - Support dependency connections between workflow nodes to specify non-data execution dependencies. v23.3.0 of DataIntegration introduces the following changes: -- The threshold field for distance measures has been improved: - - For boolean distance measures, the threshold is not shown as it has no effect. - - For normalized and unbound measures, the range of allowed values as well as an improved tooltip has been added. - - A error message is shown if the entered threshold is not valid for the given distance measure. -- Parse JSON operator now works with multiple entities. -- Updating or deleting project variables will update all affected tasks transactionally: - - The user is not allowed to delete a variable that is used in a task parameter template. - - The user is not allowed to change a variable to a value that violates restrictions in a task that uses it - - If the task parameter has a specific type (such as integer) and the template now evaluates to an incompatible type. - - If the task imposes other restrictions on a parameter (for instance, does not allow values below 0). -- base image switch to bookworm (python:3.11-slim-bookworm) -- If a path uses a property filter, an error will be thrown on writing data to a Knowledge Graph, if the property is not a valid URI. -- Support expanding all rule trees after expanding all rows in the transform evaluation. -- Change input ports definition of `Upload File to Knowledge Graph` operator from an empty fixed schema to variable inputs to make it better compatible. -- Transform page: When switching between tabs, e.g. from mapping editor to evaluation tab, the currently active rule stays active. -- JSON `#text` path now returns the formatted JSON as documented. +- The threshold field for distance measures has been improved: + - For boolean distance measures, the threshold is not shown as it has no effect. + - For normalized and unbound measures, the range of allowed values as well as an improved tooltip has been added. + - A error message is shown if the entered threshold is not valid for the given distance measure. +- Parse JSON operator now works with multiple entities. +- Updating or deleting project variables will update all affected tasks transactionally: + - The user is not allowed to delete a variable that is used in a task parameter template. + - The user is not allowed to change a variable to a value that violates restrictions in a task that uses it + - If the task parameter has a specific type (such as integer) and the template now evaluates to an incompatible type. + - If the task imposes other restrictions on a parameter (for instance, does not allow values below 0). +- base image switch to bookworm (python:3.11-slim-bookworm) +- If a path uses a property filter, an error will be thrown on writing data to a Knowledge Graph, if the property is not a valid URI. +- Support expanding all rule trees after expanding all rows in the transform evaluation. +- Change input ports definition of `Upload File to Knowledge Graph` operator from an empty fixed schema to variable inputs to make it better compatible. +- Transform page: When switching between tabs, e.g. from mapping editor to evaluation tab, the currently active rule stays active. +- JSON `#text` path now returns the formatted JSON as documented. v23.2 of DataIntegration ships the following fixes: -- Project variables widget showing the variables of the wrong project. -- Python package uninstall is not able to remove crucial packages anymore. -- Reload failed tasks after project/workspace reload. -- Reloaded failed tasks are missing the original label and description. -- Fix property pair alignment. -- Hotkey and quick search modal are not always shown on top of other modals. -- Testing for invalid documents downloaded from GDocs fixed and adapted to new behavior of wrong requests. -- Error message in 'SPARQL endpoint' plugin to mention prohibited URL redirect to a different protocol. -- `JDBC endpoint` dataset: Setting the user via JDBC URL while leaving the user parameter blank does not work. +- Project variables widget showing the variables of the wrong project. +- Python package uninstall is not able to remove crucial packages anymore. +- Reload failed tasks after project/workspace reload. +- Reloaded failed tasks are missing the original label and description. +- Fix property pair alignment. +- Hotkey and quick search modal are not always shown on top of other modals. +- Testing for invalid documents downloaded from GDocs fixed and adapted to new behavior of wrong requests. +- Error message in 'SPARQL endpoint' plugin to mention prohibited URL redirect to a different protocol. +- `JDBC endpoint` dataset: Setting the user via JDBC URL while leaving the user parameter blank does not work. ## eccenca DataManager v23.3.1 @@ -105,28 +105,28 @@ We are excited to announce the latest update to DataManager v23.3, which introdu v23.3.1 of DataManager ships the following fixes: -- _Workspace configuration:_ explore `defaultGraph` is now correctly evaluated. +- _Workspace configuration:_ explore `defaultGraph` is now correctly evaluated. v23.3.0 of DataManager adds the following new features: -- Implemented Charts module with Shacl integration. -- Added option to show edges without the shapes on the EasyNav canvas and in the sidebar, i.e. the node expansion is still shaped. -- Query module allows simple query creation with an form assisted dialogue. +- Implemented Charts module with Shacl integration. +- Added option to show edges without the shapes on the EasyNav canvas and in the sidebar, i.e. the node expansion is still shaped. +- Query module allows simple query creation with an form assisted dialogue. v23.3.0 of DataManager ships the following changes: -- Internal: - - Query module is migrated from Redux to a Context storage. - - Query module is extracted to a separate common component. -- ResourceSelect doesn't request options anymore if they have already been requested earlier. -- `shui:listQuery` allows usage of the `{{username}}` placeholder, which is replaced by the name (i.e.not the IRI) of the logged in user. +- Internal: + - Query module is migrated from Redux to a Context storage. + - Query module is extracted to a separate common component. +- ResourceSelect doesn't request options anymore if they have already been requested earlier. +- `shui:listQuery` allows usage of the `{{username}}` placeholder, which is replaced by the name (i.e.not the IRI) of the logged in user. v23.3.0 of DataManager ships the following fixes: -- Fixed broken navigation (workspace part of URL was lost). -- CMEM Manual Testing 23.2 e2e - Don't do redundant redirects in the Module context. -- Navigation tree in the Thesaurus module was collapsed after a subitem select. -- Use more space for visualization catalogue if available. +- Fixed broken navigation (workspace part of URL was lost). +- CMEM Manual Testing 23.2 e2e - Don't do redundant redirects in the Module context. +- Navigation tree in the Thesaurus module was collapsed after a subitem select. +- Use more space for visualization catalogue if available. ## eccenca DataPlatform v23.3.1 @@ -134,37 +134,37 @@ We're excited to bring you the latest update to DataPlatform v23.3, featuring nu v23.3.1 of DataPlatform ships following fixes: -- Backport of fix for URI Template ordering, when storing shaped resource + sub-resource with uriTemplate for each +- Backport of fix for URI Template ordering, when storing shaped resource + sub-resource with uriTemplate for each v23.3.0 of DataPlatform ships following fixes: -- Fixed non-working query cancelling in GraphDb 10.3 -- Wrong caching on facet query calls +- Fixed non-working query cancelling in GraphDb 10.3 +- Wrong caching on facet query calls v23.3.0 of DataPlatform adds the following new features: -- Added endpoint for removal of system resources i.e. bootstrap data -- GraphDb embedded development build +- Added endpoint for removal of system resources i.e. bootstrap data +- GraphDb embedded development build v23.3.0 of DataPlatform ships the following changes: -- Dataplatform health check update: - - activation of spring boot kubernetes health groups `liveness/readiness`. - - creation of health group `sparql` which can be (de)activated: - - `management.health.sparql.enabled`: (De)activates check from DP to store backend (default: true). - - `management.health.sparql.fixedDelayInMilliseconds`: delay in ms between store checks (default: 5000). - - `management.health.sparql.timeoutInMilliseconds`: timeout on how long to wait for store to answer check request (default: 5000). - - health group `sparql` contributes to readiness state / overall health endpoint. - - GraphDB health check uses Gdb endpoint for repository. -- Charts configuration API and Shacl integration. -- Breaking change: remove property `authorization.abox.prefix` (fixed default: ). -- Workspace configuration adjustments: - - `Application presentation` added properties companyName, applicationName, bannerBackgroundColor. - - `EasyNav module` added property shapePropertyView. - - Chart configuration module. - - Change default system workspace values. -- Spring Boot v3.1.x -- Support for the `{label}` parameter in uri templates. +- Dataplatform health check update: + - activation of spring boot kubernetes health groups `liveness/readiness`. + - creation of health group `sparql` which can be (de)activated: + - `management.health.sparql.enabled`: (De)activates check from DP to store backend (default: true). + - `management.health.sparql.fixedDelayInMilliseconds`: delay in ms between store checks (default: 5000). + - `management.health.sparql.timeoutInMilliseconds`: timeout on how long to wait for store to answer check request (default: 5000). + - health group `sparql` contributes to readiness state / overall health endpoint. + - GraphDB health check uses Gdb endpoint for repository. +- Charts configuration API and Shacl integration. +- Breaking change: remove property `authorization.abox.prefix` (fixed default: ). +- Workspace configuration adjustments: + - `Application presentation` added properties companyName, applicationName, bannerBackgroundColor. + - `EasyNav module` added property shapePropertyView. + - Chart configuration module. + - Change default system workspace values. +- Spring Boot v3.1.x +- Support for the `{label}` parameter in uri templates. ## eccenca Corporate Memory Control (cmemc) v23.3.0 @@ -172,31 +172,31 @@ We're excited to bring you the latest update to Corporate Memory Control (cmemc) v23.3.0 of Corporate Memory Control adds the following new features: -- `project variable` command group - - `create` command - create a new project variable - - `delete` command - delete a project variable - - `get` command - get the value or other data of a project variable - - `list` command - list available project variables - - `update` command - update data of an existing project variable -- `admin workspace python` command group - - `open` command - open a package pypi.org page in the browser - - `list --available` option - list published packages - - `uninstall --all` option - reset the whole python environment -- `project` command group - - `create --label` option - give a label for the created project - - `create --description` option - give a description for the created project -- `dataset` command group - - `update` command - update the configuration of an existing dataset -- `workflow` command group - - `execute --progress` option - show a progress bar -- `admin store` command group - - `bootstrap --remove` option - delete the bootstrap data +- `project variable` command group + - `create` command - create a new project variable + - `delete` command - delete a project variable + - `get` command - get the value or other data of a project variable + - `list` command - list available project variables + - `update` command - update data of an existing project variable +- `admin workspace python` command group + - `open` command - open a package pypi.org page in the browser + - `list --available` option - list published packages + - `uninstall --all` option - reset the whole python environment +- `project` command group + - `create --label` option - give a label for the created project + - `create --description` option - give a description for the created project +- `dataset` command group + - `update` command - update the configuration of an existing dataset +- `workflow` command group + - `execute --progress` option - show a progress bar +- `admin store` command group + - `bootstrap --remove` option - delete the bootstrap data v23.3.0 of Corporate Memory Control introduces the following changes: -- `workflow execute` command - more debug info when polling for workflow info -- Upgrade to `click` v8 (see Migration Notes). -- Upgrade to debian 12 based image: `3.11.6-slim-bookworm` +- `workflow execute` command - more debug info when polling for workflow info +- Upgrade to `click` v8 (see Migration Notes). +- Upgrade to debian 12 based image: `3.11.6-slim-bookworm` ## Migration Notes @@ -226,9 +226,9 @@ Due to the removal of the `authorization.abox.prefix` configuration option, a ch From v23.3 `AccessCondition`s are only regarded if their IRIs use the prefix `http://eccenca.com/` (e.g. have an IRI like `http://eccenca.com/170f25c2-3b92-40d7-b247-5bba42dbe22a`). Required action: -- If you have been using a different prefix for your `AccessCondition`s, change the prefix of these resources. E.g. by: - - search / replace the old prefix with the new one in your RDF graph backup - - using a `SPARQL query` like: +- If you have been using a different prefix for your `AccessCondition`s, change the prefix of these resources. E.g. by: + - search / replace the old prefix with the new one in your RDF graph backup + - using a `SPARQL query` like: ```sql PREFIX eccauth: @@ -262,6 +262,6 @@ From v23.3 `AccessCondition`s are only regarded if their IRIs use the prefix `ht ### cmemc -- The upgrade to `click` v8 involves new completion functions (see [completion manual](../../automate/cmemc-command-line-interface/configuration/completion-setup/index.md)) - - Old: `_CMEMC_COMPLETE=source_zsh cmemc` - - New: `_CMEMC_COMPLETE=zsh_source cmemc` +- The upgrade to `click` v8 involves new completion functions (see [completion manual](../../automate/cmemc-command-line-interface/configuration/completion-setup/index.md)) + - Old: `_CMEMC_COMPLETE=source_zsh cmemc` + - New: `_CMEMC_COMPLETE=zsh_source cmemc` diff --git a/docs/release-notes/corporate-memory-24-1/index.md b/docs/release-notes/corporate-memory-24-1/index.md index f97f7b78..99a37f5f 100644 --- a/docs/release-notes/corporate-memory-24-1/index.md +++ b/docs/release-notes/corporate-memory-24-1/index.md @@ -12,26 +12,26 @@ Corporate Memory 24.1.3 is the third patch release in the 24.1 release line. The highlights of this release are: -- Build: - - New improved **REST operator** (v2) with lots of additional features - - Extendend **Keyboard Shortcuts** in workflow editor -- Automate: - - New **`admin acl` command group** to automate management of access conditions - - New **`graph validation` command group** to automate batch validation of graph resources against SHACL shapes -- Explore and Author: - - Preview of our new **SHACL Authoring Engine** (enable with feature flag `shacl2` on your workspace configuration: `Basics`>`Workspace`>`featureFlags`) +- Build: + - New improved **REST operator** (v2) with lots of additional features + - Extendend **Keyboard Shortcuts** in workflow editor +- Automate: + - New **`admin acl` command group** to automate management of access conditions + - New **`graph validation` command group** to automate batch validation of graph resources against SHACL shapes +- Explore and Author: + - Preview of our new **SHACL Authoring Engine** (enable with feature flag `shacl2` on your workspace configuration: `Basics`>`Workspace`>`featureFlags`) This release delivers the following component versions: -- eccenca DataIntegration v24.1.1 -- eccenca DataManager v24.1.3 -- eccenca DataPlatform v24.1.2 -- eccenca Corporate Memory Control (cmemc) v24.1.4 +- eccenca DataIntegration v24.1.1 +- eccenca DataManager v24.1.3 +- eccenca DataPlatform v24.1.2 +- eccenca Corporate Memory Control (cmemc) v24.1.4 We tested this release with the following dependency components: -- Ontotext GraphDB v10.6.2 -- Keycloak v24.0.3 +- Ontotext GraphDB v10.6.2 +- Keycloak v24.0.3 More detailed information for this release is provided in the next sections. @@ -41,96 +41,96 @@ We're excited to bring you the latest update to DataIntegration v24.1, which int v24.1.2 of DataIntegration introduces the following changes: -- Description fields provide a Markdown-editor now. -- Removed Dependency ports menu item from transform and linking nodes. +- Description fields provide a Markdown-editor now. +- Removed Dependency ports menu item from transform and linking nodes. v24.1.2 of DataIntegration ships the following fixes: -- Removed bloated Amazon AWS bundle from dependencies. -- Include Snowflake JDBC driver. -- docker image: bump zlibg to mitigate CVE-2023-45853. -- docker image: remove libaom to mitigate CVE-2023-6879. -- Cannot create XML with defined DTD in output template. -- Fixed upload process in DI project files widget. +- Removed bloated Amazon AWS bundle from dependencies. +- Include Snowflake JDBC driver. +- docker image: bump zlibg to mitigate CVE-2023-45853. +- docker image: remove libaom to mitigate CVE-2023-6879. +- Cannot create XML with defined DTD in output template. +- Fixed upload process in DI project files widget. v24.1.1 of DataIntegration adds the following new features: -- A new facet has been added to the workspace search that allows to filter for read-only dataset -- The "Evaluate template" operator now supports hierarchical input entities if full evaluation is set -- Better preview of hierarchical formats, such as XML and JSON +- A new facet has been added to the workspace search that allows to filter for read-only dataset +- The "Evaluate template" operator now supports hierarchical input entities if full evaluation is set +- Better preview of hierarchical formats, such as XML and JSON v24.1.1 of DataIntegration introduces the following changes: -- Icon of notification menu was aligned to DM, it's now a bell. +- Icon of notification menu was aligned to DM, it's now a bell. v24.1.1 of DataIntegration ships the following fixes: -- Fixed various vulnerabilities -- AWS S3 workspace: IO Error Attempted read on closed stream -- Secret values (passwords) in DI task configurations not shown to users once entered -- The create project endpoint returns a custom error format instead of HTTP problem details -- Notification menu was fixed regarding its opening and closing behavior. -- XML Dataset produces wrong tags if the target property is a full URI -- Macro support for Jinja templates +- Fixed various vulnerabilities +- AWS S3 workspace: IO Error Attempted read on closed stream +- Secret values (passwords) in DI task configurations not shown to users once entered +- The create project endpoint returns a custom error format instead of HTTP problem details +- Notification menu was fixed regarding its opening and closing behavior. +- XML Dataset produces wrong tags if the target property is a full URI +- Macro support for Jinja templates v24.1.0 of DataIntegration adds the following new features: -- Multiline editing of template values -- Added loose connection of workflow nodes similar to linking editor -- XML, JSON, Excel and CSV datasets support retrieving the line and column numbers -- Error report for (validation) errors in transform and linking rule editors and transform execution report - - Shows additional details like a stacktrace and input values -- Added hotkey integration for creating new items in the workflow editor -- Improved REST operator (v2) - - With support for multiple REST requests, one per input entity - - Paging support: If the API does not return all results in a single request, this features allows to page via multiple requests and merge the results of all requests - - Better error handling and retry mechanism: Retries requests and collects errors for execution report - - Rate limiting of requests by setting a delay between subsequent requests - - Limit and offset: Only executes a specific "window" of the input entities/requests - - URL property: Allows to define a property that is injected into the result JSON that contains the original request URL - - Support dataset file output, i.e. a file based dataset can be connected to the operator output, which overwrites the dataset file with the results from the REST requests - - This allows to handle REST results as any dataset content - - Supports zip files. If a dataset (currently JSON, XML, RDF file, CSV) specifies a zip file (ending in .zip) a zip archive is written that contains one file per request result -- JSON dataset - - Support bulk resources, i.e. JSON files in a zip file - - Support reading JSON Lines files -- Python workflow plugins can now consume and produce hierarchical entities -- Additions to the workflow configuration ports: - - Allow to reconfigure transform and linking tasks in workflows - - Datasets can be connected directly to the configuration port -- Extended auto-completion support when opening the mapping (rule) editor in a workflow context: - - Support auto-completion of target properties for fixed target schema and config port schema (transformation connected to config port) - - Support auto-completion of values paths for fixed input schema -- Added timer for workflow execution and in activity view -- Error notification - - Add badge to error notification menu icon with error count. +- Multiline editing of template values +- Added loose connection of workflow nodes similar to linking editor +- XML, JSON, Excel and CSV datasets support retrieving the line and column numbers +- Error report for (validation) errors in transform and linking rule editors and transform execution report + - Shows additional details like a stacktrace and input values +- Added hotkey integration for creating new items in the workflow editor +- Improved REST operator (v2) + - With support for multiple REST requests, one per input entity + - Paging support: If the API does not return all results in a single request, this features allows to page via multiple requests and merge the results of all requests + - Better error handling and retry mechanism: Retries requests and collects errors for execution report + - Rate limiting of requests by setting a delay between subsequent requests + - Limit and offset: Only executes a specific "window" of the input entities/requests + - URL property: Allows to define a property that is injected into the result JSON that contains the original request URL + - Support dataset file output, i.e. a file based dataset can be connected to the operator output, which overwrites the dataset file with the results from the REST requests + - This allows to handle REST results as any dataset content + - Supports zip files. If a dataset (currently JSON, XML, RDF file, CSV) specifies a zip file (ending in .zip) a zip archive is written that contains one file per request result +- JSON dataset + - Support bulk resources, i.e. JSON files in a zip file + - Support reading JSON Lines files +- Python workflow plugins can now consume and produce hierarchical entities +- Additions to the workflow configuration ports: + - Allow to reconfigure transform and linking tasks in workflows + - Datasets can be connected directly to the configuration port +- Extended auto-completion support when opening the mapping (rule) editor in a workflow context: + - Support auto-completion of target properties for fixed target schema and config port schema (transformation connected to config port) + - Support auto-completion of values paths for fixed input schema +- Added timer for workflow execution and in activity view +- Error notification + - Add badge to error notification menu icon with error count. v24.1.0 of DataIntegration introduces the following changes: -- Show project variables re-ordering errors (with details) directly in project variables widget -- Support PATCH and DELETE requests in REST operators -- Upgraded libraries, in particular Play to v2.9.1 and Spark to v3.5.0 -- Support of custom tasks as input for transform and linking tasks -- Create/update dialogue: - - When a parameter value is changed that other parameters are depending on, those parameter values are reset because they might not be valid anymore -- Shortened workflow execution failure message shown in activity widget -- Added `Fail workflow` flag to `Cancel workflow` operator +- Show project variables re-ordering errors (with details) directly in project variables widget +- Support PATCH and DELETE requests in REST operators +- Upgraded libraries, in particular Play to v2.9.1 and Spark to v3.5.0 +- Support of custom tasks as input for transform and linking tasks +- Create/update dialogue: + - When a parameter value is changed that other parameters are depending on, those parameter values are reset because they might not be valid anymore +- Shortened workflow execution failure message shown in activity widget +- Added `Fail workflow` flag to `Cancel workflow` operator v24.1.0 of DataIntegration ships the following fixes: -- Many errors occurring in a form/modal, e.g. from requests, are hidden because they are shown in the global error notification which cannot be accessed while the form is open -- Missing or problematic error handling in several forms and other places -- Transform editor should show plugin labels instead of ids -- Transform execution report validation icons in mapping tree do not update after running the execution -- When upgrading a plugin, new parameters are not shown in transform editor -- Workflow editor: Creating a new connected task that has no input port connects to the output port -- Copying a project with custom prefixes into a project that misses these prefixes fails -- Workflow report always states `...has not finished execution yet.` -- Cannot add a new project variable after having tried to add it with an empty value -- Support for ARM64 architecture -- View completely crashes when error is not caught in any tab view (plugin) - there should be an error boundary -- Mapping editor shows spinner when no network is available when switching to it -- Linking editor does not load when network unavailable instead of showing error +- Many errors occurring in a form/modal, e.g. from requests, are hidden because they are shown in the global error notification which cannot be accessed while the form is open +- Missing or problematic error handling in several forms and other places +- Transform editor should show plugin labels instead of ids +- Transform execution report validation icons in mapping tree do not update after running the execution +- When upgrading a plugin, new parameters are not shown in transform editor +- Workflow editor: Creating a new connected task that has no input port connects to the output port +- Copying a project with custom prefixes into a project that misses these prefixes fails +- Workflow report always states `...has not finished execution yet.` +- Cannot add a new project variable after having tried to add it with an empty value +- Support for ARM64 architecture +- View completely crashes when error is not caught in any tab view (plugin) - there should be an error boundary +- Mapping editor shows spinner when no network is available when switching to it +- Linking editor does not load when network unavailable instead of showing error ## eccenca DataManager v24.1.3 @@ -138,49 +138,49 @@ We are excited to announce the latest update to DataManager v24.1, which introdu v24.1.3 of DataManager ships the following fixes: -- docker image: bump zlib package to mitigate CVE-2023-45853 +- docker image: bump zlib package to mitigate CVE-2023-45853 v24.1.2 of DataManager ships the following fixes: -- Version string is no longer suffixed by the dirty flag due to re-generated clients +- Version string is no longer suffixed by the dirty flag due to re-generated clients v24.1.1 of DataManager ships the following fixes: -- Inline View is used when opening a Knowledge Graph Dataset in DataIntegration -- Delete Thesaurus dialog is now working as expected -- Order of graph lists is respected, when determining the first graph to explore -- Fixed several issues with the unshaped-properties view mode of easynav, new visualizations and creating new, inverted, edges -- Hide license info for store, if no expiration date is available +- Inline View is used when opening a Knowledge Graph Dataset in DataIntegration +- Delete Thesaurus dialog is now working as expected +- Order of graph lists is respected, when determining the first graph to explore +- Fixed several issues with the unshaped-properties view mode of easynav, new visualizations and creating new, inverted, edges +- Hide license info for store, if no expiration date is available v24.1.0 of DataManager adds the following new features: -- License warnings for Corporate Memory and GraphDB license -- Added validation for invalid URI format in vocabulary registration form -- SHACL2 (beta feature, disable per default) - - support for literals - - support for object properties - - validation - - context graph -- SVG support for the object view -- A link to the DataPlatform API documentation +- License warnings for Corporate Memory and GraphDB license +- Added validation for invalid URI format in vocabulary registration form +- SHACL2 (beta feature, disable per default) + - support for literals + - support for object properties + - validation + - context graph +- SVG support for the object view +- A link to the DataPlatform API documentation v24.1.0 of DataManager ships the following changes: -- Explore Navigation Component, now supports depictions and pre-loading of the concepts list -- I18N - - Increased coverage - - Enabled nesting of the keys in translations - - Improvements in the application header in explore +- Explore Navigation Component, now supports depictions and pre-loading of the concepts list +- I18N + - Increased coverage + - Enabled nesting of the keys in translations + - Improvements in the application header in explore v24.1.0 of DataManager ships the following fixes: -- Security Update of Java wrapper -- Workspace selection resets module selection -- Considering `exploreModuleConfiguration.defaultGraph` during the Explore module mount -- Added navigation blocker for the EasyNav module -- Keeping EasyNav viewport parameters during visualization save -- Installing a vocabulary now fully refreshes the application state -- Workspaces, which are prefix of an other workspace, are now correctly handled +- Security Update of Java wrapper +- Workspace selection resets module selection +- Considering `exploreModuleConfiguration.defaultGraph` during the Explore module mount +- Added navigation blocker for the EasyNav module +- Keeping EasyNav viewport parameters during visualization save +- Installing a vocabulary now fully refreshes the application state +- Workspaces, which are prefix of an other workspace, are now correctly handled ## eccenca DataPlatform v24.1.2 @@ -188,41 +188,41 @@ We're excited to bring you the latest update to DataPlatform v24.1, which introd v24.1.2 of DataPlatform ships the following fixes: -- docker image: bump zlib package to mitigate CVE-2023-45853 +- docker image: bump zlib package to mitigate CVE-2023-45853 v24.1.1 of DataPlatform ships the following fixes: -- GraphDB license endpoints returns an empty value, if the GraphDB free is configured. +- GraphDB license endpoints returns an empty value, if the GraphDB free is configured. v24.1.0 of DataPlatform adds the following new features: -- Add license information to DataPlatform actuator info endpoint response -- Added endpoints for SHACL validation / Resource shaping - - SHACL validation and resource shaping - - endpoints for validation, node shape structure views and data retrieval - - SHACL batch validation - - added application property `scheduler.backgroundQueryPoolSize` (Default: 4) - - maximum numbers of threads for background jobs (i.e. SHACL batch validation) - - added application property `proxy.shaclBatchResultsMemoryBoundaryInMb` (Default: 100) - - amount in Megabytes (Mb) for SHACL batch validation results kept in memory for status retrieval -- Access condition review endpoint - - ability to check user rights (access conditions) for a set of groups +- Add license information to DataPlatform actuator info endpoint response +- Added endpoints for SHACL validation / Resource shaping + - SHACL validation and resource shaping + - endpoints for validation, node shape structure views and data retrieval + - SHACL batch validation + - added application property `scheduler.backgroundQueryPoolSize` (Default: 4) + - maximum numbers of threads for background jobs (i.e. SHACL batch validation) + - added application property `proxy.shaclBatchResultsMemoryBoundaryInMb` (Default: 100) + - amount in Megabytes (Mb) for SHACL batch validation results kept in memory for status retrieval +- Access condition review endpoint + - ability to check user rights (access conditions) for a set of groups v24.1.0 of DataPlatform ships the following changes: -- Static access condition prefix split for newly created access conditions - - `` – prefix for Access Condition Groups / Users - - `` – prefix for Access Conditions -- Added tracing id to audit logs -- Add feature flag field to workspace configuration -- Add support for GraphDB 10.5 +- Static access condition prefix split for newly created access conditions + - `` – prefix for Access Condition Groups / Users + - `` – prefix for Access Conditions +- Added tracing id to audit logs +- Add feature flag field to workspace configuration +- Add support for GraphDB 10.5 v24.1.0 of DataPlatform ships the following fixes: -- Allow blank nodes in update queries -- API endpoints do not return `null` values for unset fields anymore -- Correct documentation of API endpoints for named query execution -- Default language order is changed to: `["en", "", "de"]` +- Allow blank nodes in update queries +- API endpoints do not return `null` values for unset fields anymore +- Correct documentation of API endpoints for named query execution +- Default language order is changed to: `["en", "", "de"]` ## eccenca Corporate Memory Control (cmemc) v24.1.4 @@ -230,55 +230,55 @@ We're excited to bring you the latest update to Corporate Memory Control (cmemc) v24.1.4 of Corporate Memory Control (cmemc) ships the following fixes: -- restore python 3.10 compatibility +- restore python 3.10 compatibility v24.1.3 of Corporate Memory Control (cmemc) was a redacted build. v24.1.2 of Corporate Memory Control (cmemc) ships the following security patches: -- docker image: bump zlib1g to 1.3.dfsg+really1.3.1-1 to mitigate CVE-2023-45853 +- docker image: bump zlib1g to 1.3.dfsg+really1.3.1-1 to mitigate CVE-2023-45853 v24.1.1 of Corporate Memory Control (cmemc) ships the following fixes: -- In case of using env-only configuration + SSL_VERIFY=false - - InsecureRequestWarning output from urllib3 is now suppressed - - Normal user warning is given to stderr -- `admin workspace python install` command - - completion of plugin packages does not list non-plugin packages anymore +- In case of using env-only configuration + SSL_VERIFY=false + - InsecureRequestWarning output from urllib3 is now suppressed + - Normal user warning is given to stderr +- `admin workspace python install` command + - completion of plugin packages does not list non-plugin packages anymore v24.1.1 of Corporate Memory Control (cmemc) ships the following security updates: -- docker image: upgrade zlib package to 1:1.3.dfsg-3 in order to mitigate CVE-2023-45853 +- docker image: upgrade zlib package to 1:1.3.dfsg-3 in order to mitigate CVE-2023-45853 v24.1.0 of Corporate Memory Control (cmemc) adds the following new features: -- Added support for importing vocabulary from standard input (`stdin`) -- `admin acl` command group - - `create` command - Create an access condition - - `delete` command - Delete access conditions - - `inspect` command - Inspects the access condition - - `list` command - List all access conditions - - `review` command - Reviews the graph rights for a given access condition - - `update` command - Updates an access condition -- `graph validation` command group - - `execute` command - Start a validation process - - `inspect` command - Inspect validation process results - - `list` command - List validation processes - - `cancel` command - Cancel a running validation process -- `admin user list` command - - `--filter` option - filter user list -- `admin status` command - - raises an error if the Corporate Memory license is expired (grace period) - - raises a warning if the GraphDB license expires in less than one month -- `dataset create` command - - support to use JSON Lines files as JSON datasets - - support to use YAML files as TEXT datasets +- Added support for importing vocabulary from standard input (`stdin`) +- `admin acl` command group + - `create` command - Create an access condition + - `delete` command - Delete access conditions + - `inspect` command - Inspects the access condition + - `list` command - List all access conditions + - `review` command - Reviews the graph rights for a given access condition + - `update` command - Updates an access condition +- `graph validation` command group + - `execute` command - Start a validation process + - `inspect` command - Inspect validation process results + - `list` command - List validation processes + - `cancel` command - Cancel a running validation process +- `admin user list` command + - `--filter` option - filter user list +- `admin status` command + - raises an error if the Corporate Memory license is expired (grace period) + - raises a warning if the GraphDB license expires in less than one month +- `dataset create` command + - support to use JSON Lines files as JSON datasets + - support to use YAML files as TEXT datasets v24.1.0 of Corporate Memory Control (cmemc) ships the following changes: -- `graph import` command - - importing a directory to a single graph no longer raises an error but imports all turtle files to this graph -- docker image: python 3.11.8 +- `graph import` command + - importing a directory to a single graph no longer raises an error but imports all turtle files to this graph +- docker image: python 3.11.8 ## Migration Notes @@ -294,7 +294,7 @@ v24.1.0 of Corporate Memory Control (cmemc) ships the following changes: There is a known issue and existing workaround with the new dependency port feature: you may receive a message like this when running your workflows: -``` +```txt Workflow Execution Error: Not all workflow nodes were executed! Executed 2 of 7 nodes. ``` @@ -308,7 +308,7 @@ After saving it will work again. If the default workspace sets a `exploreModuleConfiguration.defaultGraph` but an additional workspace is configured to show graph lists, you might see the following error message: -``` +```txt Missing Graph configuration for Context. Please check that the graph "" actually exists. diff --git a/docs/release-notes/corporate-memory-24-2/index.md b/docs/release-notes/corporate-memory-24-2/index.md index 4c0ea926..1e2026ae 100644 --- a/docs/release-notes/corporate-memory-24-2/index.md +++ b/docs/release-notes/corporate-memory-24-2/index.md @@ -12,26 +12,26 @@ Corporate Memory 24.2.1 is the first patch release in the 24.2 release line. The highlights of this release are: -- Explore and Author: - - All-new, re-written shacl custom UI rendering engine (shacl2) is now generally available and the system default - - Support for Sankey chart type. -- Build: - - Quick creation of file based datasets in the workflow editor - dropping files into the workflow editor will automatically create a new dataset. -- Automate: - - Extension to many **import commands** to allow for importing graphs, projects, datasets and vocabularies from the web - - Extension to the **graph validation export** command to produce JUnit XML reports for better integration into CI/CD pipelines +- Explore and Author: + - All-new, re-written shacl custom UI rendering engine (shacl2) is now generally available and the system default + - Support for Sankey chart type. +- Build: + - Quick creation of file based datasets in the workflow editor - dropping files into the workflow editor will automatically create a new dataset. +- Automate: + - Extension to many **import commands** to allow for importing graphs, projects, datasets and vocabularies from the web + - Extension to the **graph validation export** command to produce JUnit XML reports for better integration into CI/CD pipelines This release delivers the following component versions: -- eccenca DataIntegration v24.2.1 -- eccenca DataManager v24.2.2 -- eccenca DataPlatform v24.2.1 -- eccenca Corporate Memory Control (cmemc) v24.2.0 +- eccenca DataIntegration v24.2.1 +- eccenca DataManager v24.2.2 +- eccenca DataPlatform v24.2.1 +- eccenca Corporate Memory Control (cmemc) v24.2.0 We tested this release with the following dependency components: -- Ontotext GraphDB v10.7.2 -- Keycloak v25.0.6 +- Ontotext GraphDB v10.7.2 +- Keycloak v25.0.6 More detailed information for this release is provided in the next sections. @@ -41,55 +41,55 @@ We're excited to bring you the latest update to DataIntegration v24.2, which int v24.2.1 of DataIntegration ships the following fixes: -- Drag and drop in react flow editors used in Linking and Transform tasks work again. -- Dragging operators in the react flow based editors when text is selected leads to large artifacts. +- Drag and drop in react flow editors used in Linking and Transform tasks work again. +- Dragging operators in the react flow based editors when text is selected leads to large artifacts. v24.2.0 of DataIntegration adds the following new features: -- Quick creation of file based datasets in the workflow editor. - - Dropping files into the workflow editor will automatically create a new dataset. -- Button to reload all cache activities at once. -- The JDBC dataset can now be configured how it will write multiple values for a single property. A new strategy allows to write multiple rows in this case. -- Python: - - Added `WorkflowContext` that allows plugins to access the workflow identifier as well as the current execution status. - - Added `packageName` attribute to the plugin JSON. +- Quick creation of file based datasets in the workflow editor. + - Dropping files into the workflow editor will automatically create a new dataset. +- Button to reload all cache activities at once. +- The JDBC dataset can now be configured how it will write multiple values for a single property. A new strategy allows to write multiple rows in this case. +- Python: + - Added `WorkflowContext` that allows plugins to access the workflow identifier as well as the current execution status. + - Added `packageName` attribute to the plugin JSON. v24.2.0 of DataIntegration introduces the following changes: -- Re-use original parameters of a replaceable dataset if the dataset type matches with the requested one in a variable workflow request. -- Support Turtle files for the graph file upload operator (No chunking supported) -- Transform object header has been separated from its properties to improve visual appearance. -- Rule editors: Add generic path operator to input path tabs. -- When trying to execute an unsaved workflow notify the user that the workflow will be saved with the option to not show the dialog again. -- Re-added reload button to data preview to get updated content on config and data changes. -- Removed URL resource manager. URLs as file names won't be resolved anymore, e.g. for dataset inputs. -- The RDF datasets will always write the schema type for each entity. +- Re-use original parameters of a replaceable dataset if the dataset type matches with the requested one in a variable workflow request. +- Support Turtle files for the graph file upload operator (No chunking supported) +- Transform object header has been separated from its properties to improve visual appearance. +- Rule editors: Add generic path operator to input path tabs. +- When trying to execute an unsaved workflow notify the user that the workflow will be saved with the option to not show the dialog again. +- Re-added reload button to data preview to get updated content on config and data changes. +- Removed URL resource manager. URLs as file names won't be resolved anymore, e.g. for dataset inputs. +- The RDF datasets will always write the schema type for each entity. v24.2.0 of DataIntegration ships the following fixes: -- Workflow Task: exception message prefixed with wrong name. -- Workflow editor: - - Workflow nodes with only a single dependency output might be executed twice. - - Loose connections from dependency ports not working anymore. - - Allow dependency connection from/to replaceable datasets. - - Drawing dependency connections from input dependency port results in unexpected connections, e.g. connections to data output ports. -- Project page breaks if file resources are missing meta data like size or modified. -- Workflow operator: became invisible after workflow error. -- Always return a 401 (not authorized) instead of a 500 response when refreshing a token has failed with an `invalid_grant` error from Keycloak. -- Rule endpoint does not return new parameters after plugin upgrade. -- Zip file created with macOS Archive Utility containing files with macOS-specific metadata not working in bulk datasets. -- Fixed `overlayEditors` that close even with unsaved changes. -- Consistent navigation behavior or indicate links/buttons that open a new tab/window. -- Replace `highlightedState` properties in workflow editor. -- Rule endpoint does not return new parameters after plugin upgrade. -- Python: Changes in plugin submodule not recognized without restarting DI. -- Dataset API is not fenced against misuse of file parameter. -- Workflow Task: exception message prefixed with wrong name. -- Improve upload icon in DI file upload widget. -- JDBC: H2 driver not found. -- JDBC: Dataset should not write an extra `rdf_type` column. -- JDBC: Cannot rerun Data preview without reopening JDBC dataset update dialog. -- JDBC: naming the URI column `uri` causes `is ambiguous` error. +- Workflow Task: exception message prefixed with wrong name. +- Workflow editor: + - Workflow nodes with only a single dependency output might be executed twice. + - Loose connections from dependency ports not working anymore. + - Allow dependency connection from/to replaceable datasets. + - Drawing dependency connections from input dependency port results in unexpected connections, e.g. connections to data output ports. +- Project page breaks if file resources are missing meta data like size or modified. +- Workflow operator: became invisible after workflow error. +- Always return a 401 (not authorized) instead of a 500 response when refreshing a token has failed with an `invalid_grant` error from Keycloak. +- Rule endpoint does not return new parameters after plugin upgrade. +- Zip file created with macOS Archive Utility containing files with macOS-specific metadata not working in bulk datasets. +- Fixed `overlayEditors` that close even with unsaved changes. +- Consistent navigation behavior or indicate links/buttons that open a new tab/window. +- Replace `highlightedState` properties in workflow editor. +- Rule endpoint does not return new parameters after plugin upgrade. +- Python: Changes in plugin submodule not recognized without restarting DI. +- Dataset API is not fenced against misuse of file parameter. +- Workflow Task: exception message prefixed with wrong name. +- Improve upload icon in DI file upload widget. +- JDBC: H2 driver not found. +- JDBC: Dataset should not write an extra `rdf_type` column. +- JDBC: Cannot rerun Data preview without reopening JDBC dataset update dialog. +- JDBC: naming the URI column `uri` causes `is ambiguous` error. ## eccenca DataManager v24.2.2 @@ -97,74 +97,74 @@ We are excited to announce the latest update to DataManager v24.2, which introdu v24.2.2 of DataManager ships the following fixes: -- Fixed the statistics display for link rules with inverted properties +- Fixed the statistics display for link rules with inverted properties v24.2.1 of DataManager ships the following fixes: -- Fixed _Create ``_-Button in explore -- Select NodeShapes according to their `sh:order` -- Resolve NodeShapes sequentially, instead of parallel -- Node selection clears its state when deleting nodes -- Easynav: non-saved values disappear after search value change -- Resource list is updated after the creation of a new resource -- Newly created relations based on inverted shapes are pointing in the right direction -- Setting language for text areas +- Fixed _Create ``_-Button in explore +- Select NodeShapes according to their `sh:order` +- Resolve NodeShapes sequentially, instead of parallel +- Node selection clears its state when deleting nodes +- Easynav: non-saved values disappear after search value change +- Resource list is updated after the creation of a new resource +- Newly created relations based on inverted shapes are pointing in the right direction +- Setting language for text areas v24.2.0 of DataManager adds the following new features: -- _Charts module_ - - Support for grouping chart series. - - Support for Sankey chart type. -- _Business Knowledge Editor - EasyNav_ - - Easynav allows to create new, directly connected resources with the via the browse dialogue. - - Improved undo/redo functionality. - - Improved internal structure and robustness. - - Improved selection and searching. - - Adding new connections via shacl shape queries. - - Adding new connections on unsaved nodes. - - Unique Visualization names are enforced. -- _Other_ - - Added a global notifications queue with the dropdown menu besides the user menu. - - Update Spring Wrapper to Spring Boot 3.2. - - Added icons for read-only graphs. - - Added "algorithm" param in workspace configuration and `/proxy/:id/resource` queries. - - Added a notification feature for retrieving the query catalog data when the backend response contains validation errors with the code "DOUBLE_TRIPLE". - - Added the multi-source turtle component. +- _Charts module_ + - Support for grouping chart series. + - Support for Sankey chart type. +- _Business Knowledge Editor - EasyNav_ + - Easynav allows to create new, directly connected resources with the via the browse dialogue. + - Improved undo/redo functionality. + - Improved internal structure and robustness. + - Improved selection and searching. + - Adding new connections via shacl shape queries. + - Adding new connections on unsaved nodes. + - Unique Visualization names are enforced. +- _Other_ + - Added a global notifications queue with the dropdown menu besides the user menu. + - Update Spring Wrapper to Spring Boot 3.2. + - Added icons for read-only graphs. + - Added "algorithm" param in workspace configuration and `/proxy/:id/resource` queries. + - Added a notification feature for retrieving the query catalog data when the backend response contains validation errors with the code "DOUBLE_TRIPLE". + - Added the multi-source turtle component. v24.2.0 of DataManager ships the following changes: -- _SHACL Component_ - SHACL2 replaces now our default SHACL viewer/editor and brings in lot of new features and enhancements. +- _SHACL Component_ - SHACL2 replaces now our default SHACL viewer/editor and brings in lot of new features and enhancements. Note: this component was already introduced in v24.1, so this list is not exhaustive. - - Improved Validation of inputs. - - Added the possibility to add properties that are not visible to shacl2. - - Migrated annotations to shacl2. - - Value and UI Query are used in relation manager, if provided. - - Added check for simple widget using `defaultResourceViewerIri`. - - Improved editor for highly connected resources. - - Improved access condition handling when creating new resources. - - Improved default language handling. - - Improved Layout for long labels. - - Partial support for qualified value shapes. - - Sticky toolbar for better usability. -- _Access Conditions_ - New Access conditions management interface replaces the former component. - - Validation for the creation items, added links to grid items. - - AC review page. + - Improved Validation of inputs. + - Added the possibility to add properties that are not visible to shacl2. + - Migrated annotations to shacl2. + - Value and UI Query are used in relation manager, if provided. + - Added check for simple widget using `defaultResourceViewerIri`. + - Improved editor for highly connected resources. + - Improved access condition handling when creating new resources. + - Improved default language handling. + - Improved Layout for long labels. + - Partial support for qualified value shapes. + - Sticky toolbar for better usability. +- _Access Conditions_ - New Access conditions management interface replaces the former component. + - Validation for the creation items, added links to grid items. + - AC review page. v24.2.0 of DataManager ships the following fixes: -- _Error messages_ - - Improved error messages format, now they are more informative and user-friendly with title and details sections. - - Warnings are shown locally where triggered and errors are added to the global notifications queue. - - Warnings are shown properly without breaking the UI, closer to the place that triggered it. -- Workflow triggers reload the page after the workflow is finished. -- Resource tags are links. -- _Explore_ - - Navigation component honors the module settings for navigationItemsPerPage. - - Navigation component shows correct pagination for search. - - Turtle tab is visible, even if the user has no write access. -- _Business Knowledge Editor - EasyNav_ - - Inverse properties are shown in the correct direction. - - Labels with more than approx. 24 characters and no white space are now correctly split into two lines. +- _Error messages_ + - Improved error messages format, now they are more informative and user-friendly with title and details sections. + - Warnings are shown locally where triggered and errors are added to the global notifications queue. + - Warnings are shown properly without breaking the UI, closer to the place that triggered it. +- Workflow triggers reload the page after the workflow is finished. +- Resource tags are links. +- _Explore_ + - Navigation component honors the module settings for navigationItemsPerPage. + - Navigation component shows correct pagination for search. + - Turtle tab is visible, even if the user has no write access. +- _Business Knowledge Editor - EasyNav_ + - Inverse properties are shown in the correct direction. + - Labels with more than approx. 24 characters and no white space are now correctly split into two lines. ## eccenca DataPlatform v24.2.1 @@ -172,60 +172,60 @@ We're excited to bring you the latest update to DataPlatform v24.2, which introd v24.2.1 of DataPlatform ships the following fixes: -- Backup archives are zipped with ZIP64 option to allow >=4G archives -- Prevent issues with long-running shacl batch jobs on single node deployments -- Include subclasses in validation of `sh:class` -- Improved validation of file uploads +- Backup archives are zipped with ZIP64 option to allow >=4G archives +- Prevent issues with long-running shacl batch jobs on single node deployments +- Include subclasses in validation of `sh:class` +- Improved validation of file uploads v24.2.0 of DataPlatform adds the following new features: -- Added migration of workspace configuration - - workspace/module configurations are stored in JSON content literals in the CMEM Config Graph since v23.1. - - New field in workspace configuration graph. - - Endpoint for migrating workspace configurations to current version. - - Extension of actuator info endpoint: Shows current version and number of items to migrate. -- Added SHACL RDF validation view for SHACL batch validations, `POST /api/shacl/validation/batches` - - `validationResultsTargetGraph`: Graph to write rdf validation model into after batch finishes - - `replace`: boolean value on whether to replace the graph (default: false) -- Added option for SHACL Batch run to query target resources with a ignore list for OWL imports, `POST /api/shacl/validation/batches` - - `owlImportsIgnoreList`: A set of graph IRIs which are not queried in the resource selection (i.e. owl imports ignored) -- Added module Access-Control to workspace configuration - - Existing module Administration split into workspace configuration and access control. - - Existing administration module used for workspace configuration (as to avoid migration steps). -- Add alternative endpoint to `/api/shapes/list` called `/api/shapes/listWithValidation` - - return object contains a field for errors in the data which prevent mapping. - - asked behavior results in shape not found responses on shapes which cannot be mapped -- Add `owlImportsResolution` to resource store endpoints - - optional parameter overriding dataplatform setting +- Added migration of workspace configuration + - workspace/module configurations are stored in JSON content literals in the CMEM Config Graph since v23.1. + - New field in workspace configuration graph. + - Endpoint for migrating workspace configurations to current version. + - Extension of actuator info endpoint: Shows current version and number of items to migrate. +- Added SHACL RDF validation view for SHACL batch validations, `POST /api/shacl/validation/batches` + - `validationResultsTargetGraph`: Graph to write rdf validation model into after batch finishes + - `replace`: boolean value on whether to replace the graph (default: false) +- Added option for SHACL Batch run to query target resources with a ignore list for OWL imports, `POST /api/shacl/validation/batches` + - `owlImportsIgnoreList`: A set of graph IRIs which are not queried in the resource selection (i.e. owl imports ignored) +- Added module Access-Control to workspace configuration + - Existing module Administration split into workspace configuration and access control. + - Existing administration module used for workspace configuration (as to avoid migration steps). +- Add alternative endpoint to `/api/shapes/list` called `/api/shapes/listWithValidation` + - return object contains a field for errors in the data which prevent mapping. + - asked behavior results in shape not found responses on shapes which cannot be mapped +- Add `owlImportsResolution` to resource store endpoints + - optional parameter overriding dataplatform setting v24.2.0 of DataPlatform ships the following changes: -- Update to Apache Jena 5 - - All JSON-LD output in JSON-LD 1.1, JSON-LD 1.0 support dropped -- Changed to Access Condition endpoints - - `GET /api/authorization/groups` return IRIs instead of names (including public/admin group) - - `GET /api/authorization splits` pageable parameter into single parameters page, size, sort -- Deactivated graph db change tracking as default -- Extension of ACL review endpoint, response updated with matching access conditions -- Additional SHUI-Property for defining object relation default view - - `shui:viewResourcesWithWidget` with values (`shui:ComplexResourceViewerWidget`, `shui:SimpleResourceViewerWidget`) -- Resource API endpoints `/proxy/{id}/resource` changed - - Additional application parameter proxy.maxCBDStatements (default: 1000000) for limiting amount of statements in memory when loading (S)CBD - - CBD calculation does not include reifications anymore - - Additional one query based algorithm for CBD calculation (algorithm can be selected with optional query parameter algorithm) - - workspace configuration: `apiConfiguration.conciseBoundLoadAlgorithm` (`ITERATIVE`, `QUERY`) -- Endpoint for workspace configuration set `/api/conf/workspaces` falls back on system default -- New default icons for resource, class and properties -- Removal of native stardog integration +- Update to Apache Jena 5 + - All JSON-LD output in JSON-LD 1.1, JSON-LD 1.0 support dropped +- Changed to Access Condition endpoints + - `GET /api/authorization/groups` return IRIs instead of names (including public/admin group) + - `GET /api/authorization splits` pageable parameter into single parameters page, size, sort +- Deactivated graph db change tracking as default +- Extension of ACL review endpoint, response updated with matching access conditions +- Additional SHUI-Property for defining object relation default view + - `shui:viewResourcesWithWidget` with values (`shui:ComplexResourceViewerWidget`, `shui:SimpleResourceViewerWidget`) +- Resource API endpoints `/proxy/{id}/resource` changed + - Additional application parameter proxy.maxCBDStatements (default: 1000000) for limiting amount of statements in memory when loading (S)CBD + - CBD calculation does not include reifications anymore + - Additional one query based algorithm for CBD calculation (algorithm can be selected with optional query parameter algorithm) + - workspace configuration: `apiConfiguration.conciseBoundLoadAlgorithm` (`ITERATIVE`, `QUERY`) +- Endpoint for workspace configuration set `/api/conf/workspaces` falls back on system default +- New default icons for resource, class and properties +- Removal of native stardog integration v24.2.0 of DataPlatform ships the following fixes: -- Broken workspace configurations fall back to system default workspace - - actuator info endpoint contains field `workspaceConfigurationError` on error -- SHACL controller validation endpoint fixed - - only validates constraints which do not need other data if change-set is given as in memory -- Fixed query rewriting of (named) graphs for users with limited read rights - - From Graphs Rewriting: Prevent rewrite of where clause if no graph variable found +- Broken workspace configurations fall back to system default workspace + - actuator info endpoint contains field `workspaceConfigurationError` on error +- SHACL controller validation endpoint fixed + - only validates constraints which do not need other data if change-set is given as in memory +- Fixed query rewriting of (named) graphs for users with limited read rights + - From Graphs Rewriting: Prevent rewrite of where clause if no graph variable found ## eccenca Corporate Memory Control (cmemc) v24.2.0 @@ -233,50 +233,50 @@ We're excited to bring you the latest update to Corporate Memory Control (cmemc) v24.2.0 of Corporate Memory Control (cmemc) adds the following new features: -- `admin store migrate` command - - Migrate configuration resources to the current version. -- `admin status` command -- - will warn in case there a workspace configurations, which can be migrated -- - will exit with exit code 1 in case option `--exit-1 always` is given and migratable workspaces are found -- `graph validation export` command - - export validation reports as JSON or jUnit XML -- `graph import` command - - support for importing graphs from remote HTTP/HTTPS locations -- `project import` command - - support for importing project zip files from remote HTTP/HTTPS locations -- `dataset create` command - - support for creation of resource file from remote HTTP/HTTPS locations -- `dataset upload` command - - support for uploading of resource file from remote HTTP/HTTPS locations -- `vocabulary import` command - - support for importing vocabulary from remote HTTP/HTTPS locations -- `smart_path` package as a replacement for `pathlib.Path` and expanded functionality to support both local file paths and remote file paths -- `ClickSmartPath` parameter type, extending `click.path` to accommodate remote files -- `graph validation execute` command group - - option `--query` to allow specifying a select query for resource selection. - - option `--ignore-graph` to provide multiple graph IRIs to be excluded from the resource selection. - - option `--result-graph` to specifies the graph where the validation results will be written. - - option `--replace` to replace the result graph with new validation results +- `admin store migrate` command + - Migrate configuration resources to the current version. +- `admin status` command + - will warn in case there a workspace configurations, which can be migrated + - will exit with exit code 1 in case option `--exit-1 always` is given and migratable workspaces are found +- `graph validation export` command + - export validation reports as JSON or jUnit XML +- `graph import` command + - support for importing graphs from remote HTTP/HTTPS locations +- `project import` command + - support for importing project zip files from remote HTTP/HTTPS locations +- `dataset create` command + - support for creation of resource file from remote HTTP/HTTPS locations +- `dataset upload` command + - support for uploading of resource file from remote HTTP/HTTPS locations +- `vocabulary import` command + - support for importing vocabulary from remote HTTP/HTTPS locations +- `smart_path` package as a replacement for `pathlib.Path` and expanded functionality to support both local file paths and remote file paths +- `ClickSmartPath` parameter type, extending `click.path` to accommodate remote files +- `graph validation execute` command group + - option `--query` to allow specifying a select query for resource selection. + - option `--ignore-graph` to provide multiple graph IRIs to be excluded from the resource selection. + - option `--result-graph` to specifies the graph where the validation results will be written. + - option `--replace` to replace the result graph with new validation results v24.2.0 of Corporate Memory Control (cmemc) ships the following fixes: -- `graph import` command - - importing a directory to a single graph no longer raises an error but imports all turtle files to this graph -- `admin workspace python install` command - - report errors from update_plugins API -- using not existing configurations (`-c` / `--configuration`) now results in a proper error message -- `workflow io` command - - can now generate ttl output files -- `admin workspace python list` command - - listing of published packages with the `--available` option now works for more than 19 packages -- `graph export` command - - newly created directories have correct access conditions now -- `vocabulary install` command - - raise proper usage error messages -- `vocabulary uninstall` command - - raise proper usage error messages -- `admin store export` command - - validates the exported zip and raises an error in case of a corrupted ZIP export +- `graph import` command + - importing a directory to a single graph no longer raises an error but imports all turtle files to this graph +- `admin workspace python install` command + - report errors from update_plugins API +- using not existing configurations (`-c` / `--configuration`) now results in a proper error message +- `workflow io` command + - can now generate ttl output files +- `admin workspace python list` command + - listing of published packages with the `--available` option now works for more than 19 packages +- `graph export` command + - newly created directories have correct access conditions now +- `vocabulary install` command + - raise proper usage error messages +- `vocabulary uninstall` command + - raise proper usage error messages +- `admin store export` command + - validates the exported zip and raises an error in case of a corrupted ZIP export ## Migration Notes @@ -308,17 +308,17 @@ jdbc:mariadb://:/?sessionVariables=sql_mode=ANSI ### DataPlatform -- Due to the update to Apache Jena 5 all JSON-LD output now conforms to JSON-LD 1.1 (was JSON-LD 1.0): - - JSON-LD 1.1 has been designed as a superset of 1.0, so we do not expect any problems, just making you aware that there _might_ be differences when using it with JSON-LD 1.0 and 1.1 processors at the same time (e.g. to compare results) +- Due to the update to Apache Jena 5 all JSON-LD output now conforms to JSON-LD 1.1 (was JSON-LD 1.0): + - JSON-LD 1.1 has been designed as a superset of 1.0, so we do not expect any problems, just making you aware that there _might_ be differences when using it with JSON-LD 1.0 and 1.1 processors at the same time (e.g. to compare results) > JSON-LD 1.1 introduces new features that are compatible with JSON-LD 1.0, but if processed by a JSON-LD 1.0 processor may produce different results (cf. [JSON-LD 1.1 Framing](https://www.w3.org/TR/json-ld11-framing/#features)) - - Affected endpoints: - - `POST /proxy/{id}/resource/framed` - - `POST /proxy/{id}/sparql/framed` - - `POST /authorization/conditions/framed` - - `POST+GET /api/queries/jsonld/perform` -- The Access Condition endpoint `GET /api/authorization/groups` returns now IRIs instead of names (including public/admin group). -- Resource API endpoints `/proxy/{id}/resource` changed, the CBD calculation does not include reifications anymore. -- The native stardog integration has been removed. As a stardog user you need to migrate your store configuration to use the generic HTTP-Store configuration option. + - Affected endpoints: + - `POST /proxy/{id}/resource/framed` + - `POST /proxy/{id}/sparql/framed` + - `POST /authorization/conditions/framed` + - `POST+GET /api/queries/jsonld/perform` +- The Access Condition endpoint `GET /api/authorization/groups` returns now IRIs instead of names (including public/admin group). +- Resource API endpoints `/proxy/{id}/resource` changed, the CBD calculation does not include reifications anymore. +- The native stardog integration has been removed. As a stardog user you need to migrate your store configuration to use the generic HTTP-Store configuration option. ### cmemc diff --git a/docs/release-notes/corporate-memory-24-3/index.md b/docs/release-notes/corporate-memory-24-3/index.md index bd11bf97..8de9c28d 100644 --- a/docs/release-notes/corporate-memory-24-3/index.md +++ b/docs/release-notes/corporate-memory-24-3/index.md @@ -12,12 +12,12 @@ Corporate Memory 24.3.2 is the third major release in 2024. The highlights of this release are: -- Explore and Author: - - New shacl2 engine now used in the Business Knowledge Editor sidebar for a whole new experience when viewing and editing node details. -- Build: - - Workflow reports now show a preview of the output entities being produced, allowing quick review and verification of the underlying workflow tasks.. -- Automate: - - The `cmem` command group `admin migration`, which adds various migration recipes to make it easier to upgrade to new versions of Corporate Memory. +- Explore and Author: + - New shacl2 engine now used in the Business Knowledge Editor sidebar for a whole new experience when viewing and editing node details. +- Build: + - Workflow reports now show a preview of the output entities being produced, allowing quick review and verification of the underlying workflow tasks.. +- Automate: + - The `cmem` command group `admin migration`, which adds various migration recipes to make it easier to upgrade to new versions of Corporate Memory. !!! info inline end "Important info" @@ -25,14 +25,14 @@ The highlights of this release are: This release delivers the following component versions: -- eccenca DataIntegration v24.3.1 -- eccenca Explore v24.3.0 (formerly DataPlatform and DataManager) -- eccenca Corporate Memory Control (cmemc) v24.3.3 +- eccenca DataIntegration v24.3.1 +- eccenca Explore v24.3.0 (formerly DataPlatform and DataManager) +- eccenca Corporate Memory Control (cmemc) v24.3.3 We tested this release with the following dependency components: -- Ontotext GraphDB v10.8.3 -- Keycloak v25.0.6 +- Ontotext GraphDB v10.8.3 +- Keycloak v25.0.6 More detailed information for this release is provided in the next sections. @@ -42,52 +42,52 @@ We're excited to bring you the latest update to DataIntegration v24.3, which int **v24.3.1 of DataIntegration adds the following new features:** -- Added download button to workflow report tab. +- Added download button to workflow report tab. **v24.3.1 of DataIntegration ships the following fixes:** -- Task descriptions with long strings do not lead to horizontal scroll bars. -- Tag search might very shortly show old search suggestions. -- Missing unit for _Matching timeout_. -- _Regex selection_ transformer has mis-formatted documentation. -- Added JDBC dataset documentation on how to configure ANSI quotes for MySQL. -- Superfluous `CREATE SILENT GRAPH` leads to slow update performance. -- Fix SQL editor inputs. +- Task descriptions with long strings do not lead to horizontal scroll bars. +- Tag search might very shortly show old search suggestions. +- Missing unit for _Matching timeout_. +- _Regex selection_ transformer has mis-formatted documentation. +- Added JDBC dataset documentation on how to configure ANSI quotes for MySQL. +- Superfluous `CREATE SILENT GRAPH` leads to slow update performance. +- Fix SQL editor inputs. **v24.3.0 of DataIntegration adds the following new features:** -- Workspace search: - - Support to filter workflows that contain replaceable datasets. - - Display tags on workflow search items when they contain replaceable datasets. - - Add file name and graph URIs to search items as searchable tags. -- Workflow editor: - - Support creating knowledge graph datasets from DataPlatform graphs matching the search query. - - Added copy prefixes option in copy task dialog. -- Integration of a Prometheus endpoint to expose many useful metrics. -- Transform operators to retrieve attributes from input tasks: - - _Input Task attributes_ retrieves individual attributes from the input task (such as the modified date) or the entire task as JSON. - - _Input file attributes_ retrieves a metadata attribute from the input file (such as the file name). -- JdbcDialect implementation for Trino: Fixes STRING type mapping, adds isolationLevel option to avoid Connections resetting AutoCommit mode and serves as example for the dialect concept. -- File hash transformer: - - Calculates the hash sum of a given file - - Works on either the input file dataset or a selected file from the project -- JSON special paths: - - `#propertyName` accesses the current object key - - `*` selects all direct children of the current token -- Add link from a task parameter description into the task's Markdown documentation for this parameter, if available. -- Show sample (output) entities for workflow operators in the workflow reports. -- Text dataset allows to configure the zip regex. -- Support setting the locale for the `Parse date pattern` and `Parse date` transform operators. - - `*` selects all direct children of the current token -- More fine-grained access control: - - In addition to a base action, it is possible to specify as many specific actions that protect specific endpoints. - - Endpoints are configured in a whitelist as URI prefixes per specific action. - - All endpoints that are protected by any specific action cannot be accessed anymore via the base action. - - Two new actions are configured by default and protect the Python plugin management and specific workspace API endpoints. See changes and migrations. -- Global variables can be marked sensitive for storing passwords: - - Sensitive variables can only be used in password fields. - - Using sensitive variables in other fields or in variable templates fails and does not expose the value. - - Example: +- Workspace search: + - Support to filter workflows that contain replaceable datasets. + - Display tags on workflow search items when they contain replaceable datasets. + - Add file name and graph URIs to search items as searchable tags. +- Workflow editor: + - Support creating knowledge graph datasets from DataPlatform graphs matching the search query. + - Added copy prefixes option in copy task dialog. +- Integration of a Prometheus endpoint to expose many useful metrics. +- Transform operators to retrieve attributes from input tasks: + - _Input Task attributes_ retrieves individual attributes from the input task (such as the modified date) or the entire task as JSON. + - _Input file attributes_ retrieves a metadata attribute from the input file (such as the file name). +- JdbcDialect implementation for Trino: Fixes STRING type mapping, adds isolationLevel option to avoid Connections resetting AutoCommit mode and serves as example for the dialect concept. +- File hash transformer: + - Calculates the hash sum of a given file + - Works on either the input file dataset or a selected file from the project +- JSON special paths: + - `#propertyName` accesses the current object key + - `*` selects all direct children of the current token +- Add link from a task parameter description into the task's Markdown documentation for this parameter, if available. +- Show sample (output) entities for workflow operators in the workflow reports. +- Text dataset allows to configure the zip regex. +- Support setting the locale for the `Parse date pattern` and `Parse date` transform operators. + - `*` selects all direct children of the current token +- More fine-grained access control: + - In addition to a base action, it is possible to specify as many specific actions that protect specific endpoints. + - Endpoints are configured in a whitelist as URI prefixes per specific action. + - All endpoints that are protected by any specific action cannot be accessed anymore via the base action. + - Two new actions are configured by default and protect the Python plugin management and specific workspace API endpoints. See changes and migrations. +- Global variables can be marked sensitive for storing passwords: + - Sensitive variables can only be used in password fields. + - Using sensitive variables in other fields or in variable templates fails and does not expose the value. + - Example: ```conf config.variables = { @@ -100,80 +100,80 @@ We're excited to bring you the latest update to DataIntegration v24.3, which int } ``` -- Delete project files operator: Allows to delete project files in a workflow based on a regex. -- Added Snowflake dataset type. +- Delete project files operator: Allows to delete project files in a workflow based on a regex. +- Added Snowflake dataset type. **v24.3.0 of DataIntegration introduces the following changes:** -- Optimized writing to Neo4j, resulting in a 25x speed improvement. -- Upgraded Spark to 3.5.3. -- Upgraded to typescript version 5.5.3. -- After saving a workflow the undo/redo queues are cleared which is consistent with other editors in DI/DM. -- Renamed DI action from `urn:eccenca:di` to ``. -- Line breaks are forced for evaluation preview tooltips. -- If a project is copied to another project, all referenced project variables and their dependent variables are copied to the target project as well. -- docker image: switch to `eclipse-temurin:17-ubi9-minimal` base image -- Prefix handling: - - Only prefixes added to a specific project are serialized/exported, no prefixes loaded by the workspace (e.g. from DP). - - Only load user prefixes and prefixes of installed vocabularies from DP into DI. -- All datasets that support zips can be written now. -- Increase visibility of breadcrumbs in application header. -- Configurable Favicon in DataIntegration. +- Optimized writing to Neo4j, resulting in a 25x speed improvement. +- Upgraded Spark to 3.5.3. +- Upgraded to typescript version 5.5.3. +- After saving a workflow the undo/redo queues are cleared which is consistent with other editors in DI/DM. +- Renamed DI action from `urn:eccenca:di` to ``. +- Line breaks are forced for evaluation preview tooltips. +- If a project is copied to another project, all referenced project variables and their dependent variables are copied to the target project as well. +- docker image: switch to `eclipse-temurin:17-ubi9-minimal` base image +- Prefix handling: + - Only prefixes added to a specific project are serialized/exported, no prefixes loaded by the workspace (e.g. from DP). + - Only load user prefixes and prefixes of installed vocabularies from DP into DI. +- All datasets that support zips can be written now. +- Increase visibility of breadcrumbs in application header. +- Configurable Favicon in DataIntegration. **v24.3.0 of DataIntegration ships the following fixes:** -- Jinja templates can lead to OutOfMemory issues. -- Loading of JDBC Type 4 Drivers from Jar at runtime. -- Add add-opens JDK option to sbt parameters to avoid Serialization errors in executors. -- User defined function removed to prevent startup error in local dev mode. -- After saving a workflow the workflow editor can be closed without warning of unsaved changes. -- Race condition in Excel map transformer cache. -- Remote Client-Side Code Execution through CSV Injection identified in penetration testing. -- CSV datasets should not be cleared at the beginning of a workflow since they are overwritten anyway. -- Ports of datasets are shown as required in workflow validation, but are not. -- In workspace/project item search disable Enter behavior while a search is pending. -- Use correct icons for copy/clone actions. -- Workflow editor: - - Workflow is not re-validated after undo/redo operations. - - Re-configuring a workflow node to not having a data output is not immediately visible (only after reload). - - When the `Create new dataset` operator is used it always creates a _dataset_ even though the item type was changed. - - Caches of file base datasets are not refreshed when updated via file download operator. - - Dependency ports checkbox does not show checkmark in workflow tasks with unconnected output port. - - Fix text on node menu options that have a checkbox. Always show the _enabled_ text. -- REST task: - - When paging is enabled and entities are output only the last request result is output. - - Add TLSv1.3 support. -- Hierarchical mapping editor: Entity relationship direction input does not show current selection. -- Transform rule editor: - - Validation errors are not shown when starting the evaluation. - - Notifications are not correctly cleared and shown. -- Transform execution report: - - Type URI validation issues are not shown in the transform execution report. - - Rule tree in transform execution report and evaluation tab has a broken collapse/expand state. -- Password parameter templates are empty initially. -- Fix issues in create/update dialog: - - Depending input gets disabled if dependent input has an empty default value. - - Data preview of dataset with nested parameters is not working. -- Task config preview has a different parameter ordering than in the create/update dialog. -- Evaluation of a text path of a text dataset in a rule editor fails. -- Cannot execute SPARQL update queries with parameter templates. -- `Evaluate template` operator: Changed project variable not updated without evaluating transform. -- Jinja interpreter does not clear previous errors. -- Process of opening and closing the handle tools menu. -- Manually defined project prefixes are automatically copied to other projects after reload. -- Removing a vocabulary does not remove the vocabulary prefix from the DI projects. -- Cannot reconfigure parameter values with templates in workflows. -- Workflow report shows multiple executions of some operators even though they were only executed once. -- Python Workflow status incorrect. -- Python Workflow operators could not be cancelled in some cases. -- Alignment dataset should support the clear method so it can be used in workflows. -- Drop zone in workflow editor freezes sometimes after dropping an operator. -- Transform/Linking operator's 'Restriction' documentation is incorrectly formatted. -- DI project "Items per page" cuts off "100" as "1...". -- Wide task descriptions are not nicely scrollable. -- Inline documentation of `Clean HTML` is incomplete/wrong. -- Cannot delete mapping rule target type anymore. -- SPARQL Construct task does not update its execution report. +- Jinja templates can lead to OutOfMemory issues. +- Loading of JDBC Type 4 Drivers from Jar at runtime. +- Add add-opens JDK option to sbt parameters to avoid Serialization errors in executors. +- User defined function removed to prevent startup error in local dev mode. +- After saving a workflow the workflow editor can be closed without warning of unsaved changes. +- Race condition in Excel map transformer cache. +- Remote Client-Side Code Execution through CSV Injection identified in penetration testing. +- CSV datasets should not be cleared at the beginning of a workflow since they are overwritten anyway. +- Ports of datasets are shown as required in workflow validation, but are not. +- In workspace/project item search disable Enter behavior while a search is pending. +- Use correct icons for copy/clone actions. +- Workflow editor: + - Workflow is not re-validated after undo/redo operations. + - Re-configuring a workflow node to not having a data output is not immediately visible (only after reload). + - When the `Create new dataset` operator is used it always creates a _dataset_ even though the item type was changed. + - Caches of file base datasets are not refreshed when updated via file download operator. + - Dependency ports checkbox does not show checkmark in workflow tasks with unconnected output port. + - Fix text on node menu options that have a checkbox. Always show the _enabled_ text. +- REST task: + - When paging is enabled and entities are output only the last request result is output. + - Add TLSv1.3 support. +- Hierarchical mapping editor: Entity relationship direction input does not show current selection. +- Transform rule editor: + - Validation errors are not shown when starting the evaluation. + - Notifications are not correctly cleared and shown. +- Transform execution report: + - Type URI validation issues are not shown in the transform execution report. + - Rule tree in transform execution report and evaluation tab has a broken collapse/expand state. +- Password parameter templates are empty initially. +- Fix issues in create/update dialog: + - Depending input gets disabled if dependent input has an empty default value. + - Data preview of dataset with nested parameters is not working. +- Task config preview has a different parameter ordering than in the create/update dialog. +- Evaluation of a text path of a text dataset in a rule editor fails. +- Cannot execute SPARQL update queries with parameter templates. +- `Evaluate template` operator: Changed project variable not updated without evaluating transform. +- Jinja interpreter does not clear previous errors. +- Process of opening and closing the handle tools menu. +- Manually defined project prefixes are automatically copied to other projects after reload. +- Removing a vocabulary does not remove the vocabulary prefix from the DI projects. +- Cannot reconfigure parameter values with templates in workflows. +- Workflow report shows multiple executions of some operators even though they were only executed once. +- Python Workflow status incorrect. +- Python Workflow operators could not be cancelled in some cases. +- Alignment dataset should support the clear method so it can be used in workflows. +- Drop zone in workflow editor freezes sometimes after dropping an operator. +- Transform/Linking operator's 'Restriction' documentation is incorrectly formatted. +- DI project "Items per page" cuts off "100" as "1...". +- Wide task descriptions are not nicely scrollable. +- Inline documentation of `Clean HTML` is incomplete/wrong. +- Cannot delete mapping rule target type anymore. +- SPARQL Construct task does not update its execution report. ## eccenca Explore v24.3.0 @@ -186,122 +186,122 @@ We are excited to announce Explore v24.3, which introduces new features, improve **v24.3.0 of Explore adds the following new features:** -- Help system - - Implemented a renovated help system with global and local context of documentation -- BusinessKnowledgeEditor (BKE) - - Rename "EasyNav" to "Business Knowledge Editor" - - Keep search bar state when visualization is saved - - Set BKE as default - - Creation of customizable class on a property shape path - - Edge type selection shows shape description on hover - - Keep search bar state when visualization is saved -- Notifications - - Added a warning message, if a user is part of a fallback admin group -- Query module - - Icons added to the query dropdown functionalities -- Access Condition - - Provided custom search function for graphs in ACDetails -- SHACL - - Creation of customizable class on a property shape path - - Workflows are triggered upon editing the resource -- Workspace configuration - - Added a support for `GRAPH` placeholder in the `navigationSearchQuery` -- Other - - Added endpoint for resolving node shapes of a resource evaluating target class only for explore - - Added flag to `/userinfo` response if user is root user - - Added support for multiline in turtle editor - - Added actuator proxy endpoint for GraphDB actuators - - hidden endpoints under `/dataplatform/actuator/proxy/graphdb/**` - - Added support for gzip payload compression in SPARQL Graph Store endpoints - - Content-Encoding / Accept-Encoding used with value gzip - - Added simple zip-bomb check for gzipped content - - Configuration: `proxy.gspUploadGzipContentLimit` sets limit in bytes of uncompressed graph file in gzip (default 5 GB) - - Added endpoint for retrieval of resource descriptions (i.e. rdfs:comment) - - signature same as for title resolving - - Added additional prometheus endpoint under different port and no authentication - - Configuration under deploy - - `deploy.additional-prometheus-endpoint.enabled` (default: false) - - `deploy.additional-prometheus-endpoint.port` (default: 9091) - - `deploy.additional-prometheus-endpoint.context` (default: /metrics) +- Help system + - Implemented a renovated help system with global and local context of documentation +- BusinessKnowledgeEditor (BKE) + - Rename "EasyNav" to "Business Knowledge Editor" + - Keep search bar state when visualization is saved + - Set BKE as default + - Creation of customizable class on a property shape path + - Edge type selection shows shape description on hover + - Keep search bar state when visualization is saved +- Notifications + - Added a warning message, if a user is part of a fallback admin group +- Query module + - Icons added to the query dropdown functionalities +- Access Condition + - Provided custom search function for graphs in ACDetails +- SHACL + - Creation of customizable class on a property shape path + - Workflows are triggered upon editing the resource +- Workspace configuration + - Added a support for `GRAPH` placeholder in the `navigationSearchQuery` +- Other + - Added endpoint for resolving node shapes of a resource evaluating target class only for explore + - Added flag to `/userinfo` response if user is root user + - Added support for multiline in turtle editor + - Added actuator proxy endpoint for GraphDB actuators + - hidden endpoints under `/dataplatform/actuator/proxy/graphdb/**` + - Added support for gzip payload compression in SPARQL Graph Store endpoints + - Content-Encoding / Accept-Encoding used with value gzip + - Added simple zip-bomb check for gzipped content + - Configuration: `proxy.gspUploadGzipContentLimit` sets limit in bytes of uncompressed graph file in gzip (default 5 GB) + - Added endpoint for retrieval of resource descriptions (i.e. rdfs:comment) + - signature same as for title resolving + - Added additional prometheus endpoint under different port and no authentication + - Configuration under deploy + - `deploy.additional-prometheus-endpoint.enabled` (default: false) + - `deploy.additional-prometheus-endpoint.port` (default: 9091) + - `deploy.additional-prometheus-endpoint.context` (default: /metrics) **v24.3.0 of Explore ships the following changes:** -- EasyNav - - Created a fallback module, marked as deprecated - - Used the old EP for saving data in easynav -- Charts Module - - Sunburst chart in the Explore module - `Statistics` tab reimplemented with ECharts -- CodeMirror editor - - Replaced the library for the common usage, added linters for the Editor -- SHACL - - Split `ShaclContextProvider` into controlled and uncontrolled versions to maintain changes from outer component - - Hardcoded descriptions endpoint replaced with a proper one from DP - - Added information about different validation types to the validation control -- `RDFResourceLinkRule` Component - - Renamed to `RDFResourceTag`, added the titles query for cases where only the item resource is provided, added the `RTKLoadingErrorElement` wrapper -- Thesaurus - - Translate the Thesaurus Module to our modern UI Stack -- Other - - docker image: switch to `eclipse-temurin:17-ubi9-minimal` base image - - Library Updates - - Spring Boot 3.3 - - Apache Jena 5.2.0 - - Removed access conditions from bootstrap data - - Add username to unauthorized graph access error in log - - Replace account information placeholders in customized queries - - Renamed `{{username}}` placeholder in the _GraphTemplateJinjaTemplate_ to `{{shuiAccountName}}` - - Renamed `{{username}}` SPARQL Query placeholder (available in `onDeleteUpdate`, `onInsertUpdate`, `onUpdateUpdate`, `shui:uiQuery` and `shui:valueQuery`) to `{{shuiAccount}}` - - Changed integration of non-validating property shapes in SHACL node shape model - - Added concept of widget integration linked to node shape which have basic SHACL Properties for form UI - - Label, Description, Order, Group, link to widget - - Widget integrations carry one of the types Workflow Trigger, Table Report or Chart - - Deprecated / Removed link from node shape to chart i.e. node shape charts are not possible anymore - - Always check GraphDb license information on `/actuator/info` call - - Changed retrieval of installed vocabulary prefixes - - Actuator info endpoint secured - - Change class hierarchy resolving to SPARQL property path instead of recursion - - Removed `ValueView` and `ValueEdit` components from resource view components group +- EasyNav + - Created a fallback module, marked as deprecated + - Used the old EP for saving data in easynav +- Charts Module + - Sunburst chart in the Explore module - `Statistics` tab reimplemented with ECharts +- CodeMirror editor + - Replaced the library for the common usage, added linters for the Editor +- SHACL + - Split `ShaclContextProvider` into controlled and uncontrolled versions to maintain changes from outer component + - Hardcoded descriptions endpoint replaced with a proper one from DP + - Added information about different validation types to the validation control +- `RDFResourceLinkRule` Component + - Renamed to `RDFResourceTag`, added the titles query for cases where only the item resource is provided, added the `RTKLoadingErrorElement` wrapper +- Thesaurus + - Translate the Thesaurus Module to our modern UI Stack +- Other + - docker image: switch to `eclipse-temurin:17-ubi9-minimal` base image + - Library Updates + - Spring Boot 3.3 + - Apache Jena 5.2.0 + - Removed access conditions from bootstrap data + - Add username to unauthorized graph access error in log + - Replace account information placeholders in customized queries + - Renamed `{{username}}` placeholder in the _GraphTemplateJinjaTemplate_ to `{{shuiAccountName}}` + - Renamed `{{username}}` SPARQL Query placeholder (available in `onDeleteUpdate`, `onInsertUpdate`, `onUpdateUpdate`, `shui:uiQuery` and `shui:valueQuery`) to `{{shuiAccount}}` + - Changed integration of non-validating property shapes in SHACL node shape model + - Added concept of widget integration linked to node shape which have basic SHACL Properties for form UI + - Label, Description, Order, Group, link to widget + - Widget integrations carry one of the types Workflow Trigger, Table Report or Chart + - Deprecated / Removed link from node shape to chart i.e. node shape charts are not possible anymore + - Always check GraphDb license information on `/actuator/info` call + - Changed retrieval of installed vocabulary prefixes + - Actuator info endpoint secured + - Change class hierarchy resolving to SPARQL property path instead of recursion + - Removed `ValueView` and `ValueEdit` components from resource view components group **v24.3.0 of Explore ships the following fixes:** -- BusinessKnowledgeEditor (BKE) - - Disabled creation of new resources via `shui:denyNewResources` property - - Added functionality to delete a resource to the node panel - - Show a notification in case of viewing details of an unshaped node - - Changed the save request payload to include a separate change for each node shape - - Added missed "remove from the canvas" functionality - - Set correct node shapes order - - Prevent Modal key event propagation - - Create new node shows node shapes instead of classes -- Turtle editor - - Cursor prevented from jumping upon error -- `MultiSourceView` Component - - Not imported warning displaying - - Prevent loosing state while navigation is triggered - - Starting with a blank resource gives a blank screen -- SHACL - - Source link from the validation log points to the correct graph - - Node shape description toggle shows markdown both when collapsed and opened - - Slow request getting the resources per node shape is replaced with the more performant one - - Tooltip on the resource list is shown correctly - - Adjusted "Add resource" disable state for simple and complex widgets - - `sh:name` shows as property shape title instead of using title helper in the dropdown - - Depictions are shown based on the vocab `foaf:depiction` property and no longer for a specific property shape - - Fetching of property values is now done purely based on pre-parameterized SPARQL queries - - Changed replacement of SHUI `{{username}}` to `{{shuiAccountName}}` - - Replace SHACL Save API - - Also use `?_graph` variables for deleting when using a value query - - Display custom `sh:message` in validation results - - Expose `sh:name` of node shape in SHACL as primary name - `rdfs:label` fallback -- Charts - - Info notification shows when query results are empty - - Context graph is conveyed to the charts query replacement -- Image widget - - The check for image widget is replaced with the regex -- Other - - Notifications: Improved rendering behavior, prevent loops - - Fixed the statistics display for link rules with inverted properties - - Exit application with code 1 on expired license +- BusinessKnowledgeEditor (BKE) + - Disabled creation of new resources via `shui:denyNewResources` property + - Added functionality to delete a resource to the node panel + - Show a notification in case of viewing details of an unshaped node + - Changed the save request payload to include a separate change for each node shape + - Added missed "remove from the canvas" functionality + - Set correct node shapes order + - Prevent Modal key event propagation + - Create new node shows node shapes instead of classes +- Turtle editor + - Cursor prevented from jumping upon error +- `MultiSourceView` Component + - Not imported warning displaying + - Prevent loosing state while navigation is triggered + - Starting with a blank resource gives a blank screen +- SHACL + - Source link from the validation log points to the correct graph + - Node shape description toggle shows markdown both when collapsed and opened + - Slow request getting the resources per node shape is replaced with the more performant one + - Tooltip on the resource list is shown correctly + - Adjusted "Add resource" disable state for simple and complex widgets + - `sh:name` shows as property shape title instead of using title helper in the dropdown + - Depictions are shown based on the vocab `foaf:depiction` property and no longer for a specific property shape + - Fetching of property values is now done purely based on pre-parameterized SPARQL queries + - Changed replacement of SHUI `{{username}}` to `{{shuiAccountName}}` + - Replace SHACL Save API + - Also use `?_graph` variables for deleting when using a value query + - Display custom `sh:message` in validation results + - Expose `sh:name` of node shape in SHACL as primary name - `rdfs:label` fallback +- Charts + - Info notification shows when query results are empty + - Context graph is conveyed to the charts query replacement +- Image widget + - The check for image widget is replaced with the regex +- Other + - Notifications: Improved rendering behavior, prevent loops + - Fixed the statistics display for link rules with inverted properties + - Exit application with code 1 on expired license ## eccenca Corporate Memory Control (cmemc) v24.3.3 @@ -309,78 +309,78 @@ We're excited to bring you the latest update to Corporate Memory Control (cmemc) **v24.3.3 of cmemc provides the following fixes:** -- add missing migration recipe for deprecated SPARQL datatypes +- add missing migration recipe for deprecated SPARQL datatypes **v24.3.2 of cmemc provides the following fixes:** -- remove accidentally added pip dependency +- remove accidentally added pip dependency **v24.3.1 of cmemc provides the following fixes:** -- `graph import` command - - use python stdlib instead rdflib to guess mime types (lower memory footprint) +- `graph import` command + - use python stdlib instead rdflib to guess mime types (lower memory footprint) **v24.3.0 of cmemc adds the following new features:** -- `graph validation execute` command - - `--inspect` option to return the list of violations instead of the summary (includes `--wait`) -- `graph validation inpect` command - - retrieval and display of titles as terminal links for resources - - completion: retrieval and display of titles as descriptions -- `graph validation list` command - - retrieval and display of titles as terminal links for graphs -- `graph export` command - - option `--compress` to generate compressed ttl file -- `graph import` command - - support import of compressed ttl/nt files -- `admin store export` command - - `--replace` option to replace an existing file - - if no BACKUP_FILE is given, a default of `{{date}}-{{connection}}.store.zip` is used -- `project import` command - - `--replace` option to replace an existing project -- `project export` command - - `--replace` option to replace an existing file -- `admin workspace export` - - `--replace` option to replace an existing file -- `admin metrics` command group - - support for build / data integration metrics, e.g. `build:cmem_workspace_task_spec_size` - - support for GraphDB store metrics, e.g. `store:graphdb_slow_queries_count` -- `admin metrics list` command - - documentation column to output table - - `--filter` option to filter metrics table by job, name, ID, or type -- `admin acl` command group - - support for updated 24.3 access condition vocabulary and ACL graph -- `admin migration` command group - - `admin migration list` command - List migration recipes - - `admin migration execute` command - Execute needed migration recipes - - The following migration recipes are available: - - `bootstrap-data` - Re-import bootstrap system data to match current version - - `workspace-configurations` - Forward-upgrade explore workspace configurations - - `acl-graph-24.3` - Move access conditions and used queries to new ACL graph - - `acl-vocab-24.3` - Migrate auth vocabulary terms (actions and other grants) - - `chart-widgets-24.3` - Migrate Chart Property Shapes to Widget Integrations - - `workflow-trigger-widgets-24.3` - Migrate Workflow Trigger Property Shapes to Widget Integrations +- `graph validation execute` command + - `--inspect` option to return the list of violations instead of the summary (includes `--wait`) +- `graph validation inpect` command + - retrieval and display of titles as terminal links for resources + - completion: retrieval and display of titles as descriptions +- `graph validation list` command + - retrieval and display of titles as terminal links for graphs +- `graph export` command + - option `--compress` to generate compressed ttl file +- `graph import` command + - support import of compressed ttl/nt files +- `admin store export` command + - `--replace` option to replace an existing file + - if no BACKUP_FILE is given, a default of `{{date}}-{{connection}}.store.zip` is used +- `project import` command + - `--replace` option to replace an existing project +- `project export` command + - `--replace` option to replace an existing file +- `admin workspace export` + - `--replace` option to replace an existing file +- `admin metrics` command group + - support for build / data integration metrics, e.g. `build:cmem_workspace_task_spec_size` + - support for GraphDB store metrics, e.g. `store:graphdb_slow_queries_count` +- `admin metrics list` command + - documentation column to output table + - `--filter` option to filter metrics table by job, name, ID, or type +- `admin acl` command group + - support for updated 24.3 access condition vocabulary and ACL graph +- `admin migration` command group + - `admin migration list` command - List migration recipes + - `admin migration execute` command - Execute needed migration recipes + - The following migration recipes are available: + - `bootstrap-data` - Re-import bootstrap system data to match current version + - `workspace-configurations` - Forward-upgrade explore workspace configurations + - `acl-graph-24.3` - Move access conditions and used queries to new ACL graph + - `acl-vocab-24.3` - Migrate auth vocabulary terms (actions and other grants) + - `chart-widgets-24.3` - Migrate Chart Property Shapes to Widget Integrations + - `workflow-trigger-widgets-24.3` - Migrate Workflow Trigger Property Shapes to Widget Integrations **In addition to that, these changes and fixes are included:** -- cmemc will not fail anymore when the config dir is not creatable (message in debug) -- cmemc will not fail anymore when the config ini is not readable (message in debug) -- For these commands `admin acl list`, `dataset list`, `graph list`, `project list`, `admin user list`, `project variable list`, `vocabulary list`, `workflow list`, `admin workspace python list`, `admin workspace python list-plugins`, `dataset resource list`, `workflow scheduler list`, and `vocabulary cache list`: - - ommit empty tables with usage note message -- `admin status` command - - component name change: DI -> BUILD - - component name change: DP -> EXPLORE - - component removal: DM (merged with DP into EXPLORE) - - key prefix change: dp -> explore - - key prefix change: di -> build -- `project export` command - - `--filename-template` completion examples adaption -- `dataset create` command - - Support compressed zip files for dataset types including CSV, XML, JSON, YAML, and plain text. -- `admin metrics` command group - - metrics identification now as combined ID of `job_id:metrics_name` -- `admin metrics` command group - - `--job` option, use `--filter job job_id` or combined metrics ID instead +- cmemc will not fail anymore when the config dir is not creatable (message in debug) +- cmemc will not fail anymore when the config ini is not readable (message in debug) +- For these commands `admin acl list`, `dataset list`, `graph list`, `project list`, `admin user list`, `project variable list`, `vocabulary list`, `workflow list`, `admin workspace python list`, `admin workspace python list-plugins`, `dataset resource list`, `workflow scheduler list`, and `vocabulary cache list`: + - ommit empty tables with usage note message +- `admin status` command + - component name change: DI -> BUILD + - component name change: DP -> EXPLORE + - component removal: DM (merged with DP into EXPLORE) + - key prefix change: dp -> explore + - key prefix change: di -> build +- `project export` command + - `--filename-template` completion examples adaption +- `dataset create` command + - Support compressed zip files for dataset types including CSV, XML, JSON, YAML, and plain text. +- `admin metrics` command group + - metrics identification now as combined ID of `job_id:metrics_name` +- `admin metrics` command group + - `--job` option, use `--filter job job_id` or combined metrics ID instead ## Migration Notes @@ -394,12 +394,12 @@ We're excited to bring you the latest update to Corporate Memory Control (cmemc) ### eccenca DataIntegration -- CSV files are no longer deleted by default at the beginning of a workflow execution. This behavior can be changed in the CSV dataset configuration. -- Access control changes. Action URIs have been renamed and new actions are introduced by default: - - `urn:eccenca:di` -> `` (will be handled by `cmemc admin migration`, see below). - - `urn:elds-backend-all-actions` -> `` (will be handled by `cmemc admin migration`, see below). - - Python plugin management endpoints are now secured via `` action. - - Workspace admin functions (reload workspace, import workspace) are now secured via `` action. +- CSV files are no longer deleted by default at the beginning of a workflow execution. This behavior can be changed in the CSV dataset configuration. +- Access control changes. Action URIs have been renamed and new actions are introduced by default: + - `urn:eccenca:di` -> `` (will be handled by `cmemc admin migration`, see below). + - `urn:elds-backend-all-actions` -> `` (will be handled by `cmemc admin migration`, see below). + - Python plugin management endpoints are now secured via `` action. + - Workspace admin functions (reload workspace, import workspace) are now secured via `` action. ### eccenca Explore @@ -420,17 +420,17 @@ Run all migrations: `cmemc admin migration execute --all`. ### eccenca Corporate Memory Control (cmemc) -- All scripts which used the `admin status` command with the `--key` option: - - adapt the key prefixes accordingly: - - old: `cmemc admin status --key dp.info.license.validDate` - - new: `cmemc admin status --key explore.info.license.validDate` -- `admin store migrate` command deprecated - - use the `admin migration` command group instead -- `--overwrite` options deprecated - will be removed with the next major version - - affected commands: - - `project import` command - - `project export` command - - `admin workspace export` command -- All scripts which used the `admin metrics` command group: - - use combined metrics ID of `job_id:metrics_name` - - use `--filter job job_id` instead of `--job job_id` +- All scripts which used the `admin status` command with the `--key` option: + - adapt the key prefixes accordingly: + - old: `cmemc admin status --key dp.info.license.validDate` + - new: `cmemc admin status --key explore.info.license.validDate` +- `admin store migrate` command deprecated + - use the `admin migration` command group instead +- `--overwrite` options deprecated - will be removed with the next major version + - affected commands: + - `project import` command + - `project export` command + - `admin workspace export` command +- All scripts which used the `admin metrics` command group: + - use combined metrics ID of `job_id:metrics_name` + - use `--filter job job_id` instead of `--job job_id` diff --git a/docs/release-notes/corporate-memory-25-1/index.md b/docs/release-notes/corporate-memory-25-1/index.md index 7c207eeb..113123cb 100644 --- a/docs/release-notes/corporate-memory-25-1/index.md +++ b/docs/release-notes/corporate-memory-25-1/index.md @@ -12,17 +12,17 @@ Corporate Memory 25.1 is the first major release in 2025. The highlights of this release are: -- Build: **Seamless Workflow Integration** - - Directly connecting datasets with explicit schemas to workflow operators simplifies data ingestion and processing, allowing users to quickly incorporate CSV and text data into their workflows. +- Build: **Seamless Workflow Integration** + - Directly connecting datasets with explicit schemas to workflow operators simplifies data ingestion and processing, allowing users to quickly incorporate CSV and text data into their workflows. -- Build: **Improved Rule Editing Experience** - - Enhanced copy & paste functionality in rule editors boosts productivity by making it easier to manage and edit rules accurately and efficiently. +- Build: **Improved Rule Editing Experience** + - Enhanced copy & paste functionality in rule editors boosts productivity by making it easier to manage and edit rules accurately and efficiently. -- Explore and Autor: **Streamlined Shape Management** - - The introduction of new SHACL shape quick-access options empowers users to effortlessly build, validate, and troubleshoot complex shape configuration. +- Explore and Autor: **Streamlined Shape Management** + - The introduction of new SHACL shape quick-access options empowers users to effortlessly build, validate, and troubleshoot complex shape configuration. -- Automate: **Lightning-fast Parameterized Queries** - - The new `cmemc` query placeholder specifications enable super-fast execution of parameterized queries by running background value queries to provide dynamic completions, significantly enhancing data query responsiveness. +- Automate: **Lightning-fast Parameterized Queries** + - The new `cmemc` query placeholder specifications enable super-fast execution of parameterized queries by running background value queries to provide dynamic completions, significantly enhancing data query responsiveness. !!! info inline end "Important info" @@ -30,14 +30,14 @@ The highlights of this release are: This release delivers the following component versions: -- eccenca DataIntegration v25.1.1 -- eccenca Explore v25.1.2 (formerly DataPlatform and DataManager) -- eccenca Corporate Memory Control (cmemc) v25.1.1 +- eccenca DataIntegration v25.1.1 +- eccenca Explore v25.1.2 (formerly DataPlatform and DataManager) +- eccenca Corporate Memory Control (cmemc) v25.1.1 We tested this release with the following dependency components: -- Ontotext GraphDB v10.8.3 -- Keycloak v25.0.6 +- Ontotext GraphDB v10.8.3 +- Keycloak v25.0.6 More detailed information for this release is provided in the next sections. @@ -48,46 +48,46 @@ We're excited to bring you the latest update to DataIntegration v25.1, which int **v25.1.1 of DataIntegration ships the following fixes and additions:** - OIDC - - added support for Request Party-Initiated Logout as specified by OpenID Connect + - added support for Request Party-Initiated Logout as specified by OpenID Connect - S3 - - Now defaults to the AWS credentials provider chain if S3 access and secret keys are not configured. + - Now defaults to the AWS credentials provider chain if S3 access and secret keys are not configured. - Python - - Prevented redundant re-imports of already loaded modules. - - Updated `FileEntitySchema` in `cmem-plugin-base` for improved compatibility with datasets. + - Prevented redundant re-imports of already loaded modules. + - Updated `FileEntitySchema` in `cmem-plugin-base` for improved compatibility with datasets. - RegexExtractionTransformer - - Resolved an issue that incorrectly generated null values. + - Resolved an issue that incorrectly generated null values. **v25.1.0 of DataIntegration adds the following new features:** -- Use colors for workbench tags. -- Added a new operator for concatenating input values into a file. -- Enabled copy & paste functionality in rule editors. -- Datasets with explicit schemas can now be directly connected to workflow operators. - - Supported for CSV and text datasets. - - If a supported dataset is connected to a workflow operator with a flexible input schema, the entire dataset (i.e., all properties of its primary type) is read. - - For CSV datasets, this results in entities being read with all columns included. -- Allow changing the width of blocks in the mapping editor. +- Use colors for workbench tags. +- Added a new operator for concatenating input values into a file. +- Enabled copy & paste functionality in rule editors. +- Datasets with explicit schemas can now be directly connected to workflow operators. + - Supported for CSV and text datasets. + - If a supported dataset is connected to a workflow operator with a flexible input schema, the entire dataset (i.e., all properties of its primary type) is read. + - For CSV datasets, this results in entities being read with all columns included. +- Allow changing the width of blocks in the mapping editor. **v25.1.0 of DataIntegration introduces the following changes:** -- Invisible parameters are now part of the config port schema. -- Improved file names for downloaded projects and workspaces. -- SPARQL results are streamed as JSON instead of XML. -- The root breadcrumb and the _Build_ logo in the navigation sidebar now direct to the _projects_ search facet instead of _All types_. +- Invisible parameters are now part of the config port schema. +- Improved file names for downloaded projects and workspaces. +- SPARQL results are streamed as JSON instead of XML. +- The root breadcrumb and the _Build_ logo in the navigation sidebar now direct to the _projects_ search facet instead of _All types_. **v25.1.0 of DataIntegration ships the following fixes:** -- Fixed URI rule evaluation failure for empty object mappings. -- No duplicate JDBC jar configuration is required anymore. -- Fixed issue with JSON datasets not always navigating into arrays. -- Fixed issue where direct transform execution does not use project variables. -- Fixed Transform Evaluation failure when a rule contains a template transformer. -- Fixed issue where URI pattern input sometimes resets to its initial value or crashes the mapping editor. -- Fixed issue where SPARQL restriction expands the wrong SPARQL pattern when using property paths with prefixed names. -- Fixed RDF file upload issue. -- Fixed issue where the reference entities cache fails to load a large number of entities from the RDF store. -- Fixed issue where tasks created in the workflow editor are not added to the recently viewed list. -- Fixed issue where adding a note to a linking rule fails to save. +- Fixed URI rule evaluation failure for empty object mappings. +- No duplicate JDBC jar configuration is required anymore. +- Fixed issue with JSON datasets not always navigating into arrays. +- Fixed issue where direct transform execution does not use project variables. +- Fixed Transform Evaluation failure when a rule contains a template transformer. +- Fixed issue where URI pattern input sometimes resets to its initial value or crashes the mapping editor. +- Fixed issue where SPARQL restriction expands the wrong SPARQL pattern when using property paths with prefixed names. +- Fixed RDF file upload issue. +- Fixed issue where the reference entities cache fails to load a large number of entities from the RDF store. +- Fixed issue where tasks created in the workflow editor are not added to the recently viewed list. +- Fixed issue where adding a note to a linking rule fails to save. ## eccenca Explore v25.1.2 @@ -100,79 +100,79 @@ We are excited to announce Explore v25.1, which introduces new features, improve **v25.1.2 of Explore ships the following fixes:** -- Spring Boot Gradle plugin patch upgrade to address CVE-2025-31651 -- Consider owl import resolution in QUERY CBD resolution strategy -- Link Rules - Rule Setup: parameters of paths retain values, changes on save are shown, even if requests in the background still run. -- Fix ACL Management rights for writing access conditions +- Spring Boot Gradle plugin patch upgrade to address CVE-2025-31651 +- Consider owl import resolution in QUERY CBD resolution strategy +- Link Rules - Rule Setup: parameters of paths retain values, changes on save are shown, even if requests in the background still run. +- Fix ACL Management rights for writing access conditions **v25.1.1 of Explore ships the following fixes:** -- OIDC - - Add deployment property for post logout redirect uri -- Shacl - - Disable adding properties if the max amount of properties is reached - - Show fields of subshapes -- BKE - - Prevent loading candidates queries for readonly properties - - Switching between nodeshapes +- OIDC + - Add deployment property for post logout redirect uri +- Shacl + - Disable adding properties if the max amount of properties is reached + - Show fields of subshapes +- BKE + - Prevent loading candidates queries for readonly properties + - Switching between nodeshapes **v25.1.0 of Explore adds the following new features:** -- Other - - Added support for Virtuoso 8.3: - - Uses the eccenca Docker image for GitLab CI tests. - - Includes adjustments in the store connection to address specific Virtuoso issues. -- SHACL - - Added a download option for value queries in the complex view. - - Values in the table view are now sorted by IRI by default; this can be overridden by setting `shui:disableDefaultValueSorting true`. - - Added a new SHACL form to the graph creation interface. - - Added a debug node shape option for quick access. - - Corrected the display of lists of `xsd:anyURI` literals with long URIs. -- BKE - - Improved whitespace formatting in the BKE dossier. +- Other + - Added support for Virtuoso 8.3: + - Uses the eccenca Docker image for GitLab CI tests. + - Includes adjustments in the store connection to address specific Virtuoso issues. +- SHACL + - Added a download option for value queries in the complex view. + - Values in the table view are now sorted by IRI by default; this can be overridden by setting `shui:disableDefaultValueSorting true`. + - Added a new SHACL form to the graph creation interface. + - Added a debug node shape option for quick access. + - Corrected the display of lists of `xsd:anyURI` literals with long URIs. +- BKE + - Improved whitespace formatting in the BKE dossier. **v25.1.0 of Explore ships the following changes:** -- SHACL - - Conditionally hid the remove, create, and clone buttons. - - Added support for GraphDB 10.8.3. - - Removed quad upload support for GSP and the upload endpoint +- SHACL + - Conditionally hid the remove, create, and clone buttons. + - Added support for GraphDB 10.8.3. + - Removed quad upload support for GSP and the upload endpoint (GSP quads are not supported by stores or are uploaded as triples to a single graph only). -- Link Rules - - Adjusted link rules to use the new ACL API. -- SHACL - - Stabilized the UI during loading. - - Updated graph creation forms to the current SHACL system. -- BKE - - Merged the display of relations when property shape mode is deactivated. - - Saved graph changes while preserving the visualization state. -- Query Module - - Catalogue queries are now deleted using resource deletion (CBD). -- Other - - Switched the backend build system to use _Maven Central_ instead of _Artifactory_, which also removes the blocking Virtuoso dependency. - - Added `POST` endpoints for `GET` data requests that may result in long IRIs. - - Updated to Spring Boot 3.4. - - Made Apache Jena SPARQL query result streaming adjustable via the `proxy.proxy-sparql-streaming-format` configuration (default: `XML`; possible values: `JSON`, `XML`). - - Updated Apache Jena to version 5.3.0. - - Implemented a new serialization method for paged responses (currently relevant only for access condition management endpoints/clients). +- Link Rules + - Adjusted link rules to use the new ACL API. +- SHACL + - Stabilized the UI during loading. + - Updated graph creation forms to the current SHACL system. +- BKE + - Merged the display of relations when property shape mode is deactivated. + - Saved graph changes while preserving the visualization state. +- Query Module + - Catalogue queries are now deleted using resource deletion (CBD). +- Other + - Switched the backend build system to use _Maven Central_ instead of _Artifactory_, which also removes the blocking Virtuoso dependency. + - Added `POST` endpoints for `GET` data requests that may result in long IRIs. + - Updated to Spring Boot 3.4. + - Made Apache Jena SPARQL query result streaming adjustable via the `proxy.proxy-sparql-streaming-format` configuration (default: `XML`; possible values: `JSON`, `XML`). + - Updated Apache Jena to version 5.3.0. + - Implemented a new serialization method for paged responses (currently relevant only for access condition management endpoints/clients). **v25.1.0 of Explore ships the following fixes:** -- Other - - Render node shapes without property shapes correctly—that is, display their widgets. - - The root admin username now resolves to the actual account name rather than a fixed `admin`. - - GSP file uploads via multipart requests now allow file suffixes in uppercase. - - Re-enabled Prometheus cache metrics. - - URIs with escaped characters are now preserved. - - Added an indication for broken workspace configurations in the UI. - - Fixed missing translations in messages prompting necessary re-login. -- SHACL - - Resolved the `shuiObject` placeholder correctly in custom queries when a resource is created. - - In Shacline, subshapes now have a cutoff of 20; a warning is displayed when the limit is reached. - - Fixed an issue where adding a new subshape caused unwanted duplication of existing subshapes of the same type. - - Subshapes are now removed if their removal is revoked. -- BusinessKnowledgeEditor (BKE) - - Added support for value queries in BKE. +- Other + - Render node shapes without property shapes correctly—that is, display their widgets. + - The root admin username now resolves to the actual account name rather than a fixed `admin`. + - GSP file uploads via multipart requests now allow file suffixes in uppercase. + - Re-enabled Prometheus cache metrics. + - URIs with escaped characters are now preserved. + - Added an indication for broken workspace configurations in the UI. + - Fixed missing translations in messages prompting necessary re-login. +- SHACL + - Resolved the `shuiObject` placeholder correctly in custom queries when a resource is created. + - In Shacline, subshapes now have a cutoff of 20; a warning is displayed when the limit is reached. + - Fixed an issue where adding a new subshape caused unwanted duplication of existing subshapes of the same type. + - Subshapes are now removed if their removal is revoked. +- BusinessKnowledgeEditor (BKE) + - Added support for value queries in BKE. ## eccenca Corporate Memory Control (cmemc) v25.1.1 @@ -180,40 +180,39 @@ We're excited to bring you the latest update to Corporate Memory Control (cmemc) **v25.1.1 of cmemc introduces the following changes:** -- corrected target versions -- corrected migration target versions +- corrected target versions +- corrected migration target versions **v25.1.0 of cmemc adds the following new features and change behaviour:** -- `query execute` command - - shell completion of placeholder values (using annotated QueryPlaceholder resources) -- `admin workspace python reload` command - - reload / register all installed plugins into the DataIntegration workspace -- `admin workspace python list-plugins` command - - will warn now if plugins are installed but not registered -- `admin migration` command group - - `hide-header-footer-25.1` migration recipe - - Remove triples using deprecated shui:valueQueryHideHeader|Footer terms -- `query execute` command - - in case the user does not request a specific content type, some results are shown as a table (instead of `text/csv`) - - `--accept` option now has completion support -- base command - - `--external-http-timeout` option to specify the timeout for non-CMEM HTTP requests -- configuration via INI config file - - allow debug and proxy settings for a connection - - allow settings in the `DEFAULT` section of the config file for all connections -- change in configuration loading order, to integrate the values from the `DEFAULT` section: - 1. load environment variables into options-dict (click is doing this for `CMEMC_` variables) - 2. load options from command line and overwrite environment (only `CMEMC_` variables) - 3. load `DEFAULT` value keys, but not for keys which are already set (i.e not override cli options or env variables) - 4. load named INI section values (in case there is an INI section given) -> this will not overwrite everything - 5. use API defaults if there are not enough config keys (use default `CMEM_BASE_URI`, default `OAUTH_GRANT_TYPE` and default `OAUTH_CLIENT_ID`|`SECRET` if not present) - +- `query execute` command + - shell completion of placeholder values (using annotated QueryPlaceholder resources) +- `admin workspace python reload` command + - reload / register all installed plugins into the DataIntegration workspace +- `admin workspace python list-plugins` command + - will warn now if plugins are installed but not registered +- `admin migration` command group + - `hide-header-footer-25.1` migration recipe + - Remove triples using deprecated shui:valueQueryHideHeader|Footer terms +- `query execute` command + - in case the user does not request a specific content type, some results are shown as a table (instead of `text/csv`) + - `--accept` option now has completion support +- base command + - `--external-http-timeout` option to specify the timeout for non-CMEM HTTP requests +- configuration via INI config file + - allow debug and proxy settings for a connection + - allow settings in the `DEFAULT` section of the config file for all connections +- change in configuration loading order, to integrate the values from the `DEFAULT` section: + 1. load environment variables into options-dict (click is doing this for `CMEMC_` variables) + 2. load options from command line and overwrite environment (only `CMEMC_` variables) + 3. load `DEFAULT` value keys, but not for keys which are already set (i.e not override cli options or env variables) + 4. load named INI section values (in case there is an INI section given) -> this will not overwrite everything + 5. use API defaults if there are not enough config keys (use default `CMEM_BASE_URI`, default `OAUTH_GRANT_TYPE` and default `OAUTH_CLIENT_ID`|`SECRET` if not present) **In addition the following changes and fixes are included:** -- `admin workspace python uninstall` command - - shell completion uses correct connection now +- `admin workspace python uninstall` command + - shell completion uses correct connection now ## Migration Notes @@ -232,5 +231,5 @@ We're excited to bring you the latest update to Corporate Memory Control (cmemc) ### eccenca Corporate Memory Control (cmemc) -- `query execute` command - - use `--accept` in case you need explicit CSV output +- `query execute` command + - use `--accept` in case you need explicit CSV output diff --git a/docs/release-notes/corporate-memory-25-2/index.md b/docs/release-notes/corporate-memory-25-2/index.md index b8ea23bc..9f1fa6ca 100644 --- a/docs/release-notes/corporate-memory-25-2/index.md +++ b/docs/release-notes/corporate-memory-25-2/index.md @@ -14,17 +14,17 @@ Corporate Memory 25.2 is the second major release in 2025. The highlights of this release are: -- Build: **Enhanced File Management in Workflows** - - New binary file dataset and project file operators enable seamless integration of PDFs, images, and other binary files directly into workflows, streamlining document processing pipelines. +- Build: **Enhanced File Management in Workflows** + - New binary file dataset and project file operators enable seamless integration of PDFs, images, and other binary files directly into workflows, streamlining document processing pipelines. -- Explore: **Dynamic Class and Property Creation** - - Create classes and properties on-the-fly while defining SHACL shapes, dramatically accelerating ontology development and data modeling workflows without context switching. +- Explore: **Dynamic Class and Property Creation** + - Create classes and properties on-the-fly while defining SHACL shapes, dramatically accelerating ontology development and data modeling workflows without context switching. -- Explore and Automate: **Multi-Graph Query Management** - - The enhanced query catalog now supports multiple query graphs and arbitrary graph selection, enabling better organization and management of SPARQL queries across different knowledge domains. +- Explore and Automate: **Multi-Graph Query Management** + - The enhanced query catalog now supports multiple query graphs and arbitrary graph selection, enabling better organization and management of SPARQL queries across different knowledge domains. -- Build: **Mapping Creator** (BETA) - - New visual mapping management and GenAI based mapping environment, allowing unparalleled clarity, speed and ease in building and maintaining your mapping rules. +- Build: **Mapping Creator** (BETA) + - New visual mapping management and GenAI based mapping environment, allowing unparalleled clarity, speed and ease in building and maintaining your mapping rules. !!! info inline end "Important info" @@ -32,14 +32,14 @@ The highlights of this release are: This release delivers the following component versions: -- eccenca DataIntegration v25.2.2 -- eccenca Explore v25.2.6 -- eccenca Corporate Memory Control (cmemc) v25.4.0 +- eccenca DataIntegration v25.2.2 +- eccenca Explore v25.2.6 +- eccenca Corporate Memory Control (cmemc) v25.4.0 We tested this release with the following dependency components: -- Ontotext GraphDB v11.0.2 -- Keycloak v25.0.6 +- Ontotext GraphDB v11.0.2 +- Keycloak v25.0.6 More detailed information for this release is provided in the next sections. @@ -49,49 +49,49 @@ We are excited to announce the release of DataIntegration v25.2.2, which introdu **v25.2.2 of DataIntegration ships the following fixes and additions:** -- Added optional parameter to **Get project files** operator to set the MIME type of the retrieved files, required for uploading RDF files to a Knowledge Graph dataset +- Added optional parameter to **Get project files** operator to set the MIME type of the retrieved files, required for uploading RDF files to a Knowledge Graph dataset **v25.2.1 of DataIntegration ships the following fixes and additions:** -- Icon button tooltips do not pop up out of control -- Edge markers are displayed in workflow editor -- If a project is imported to the knowledge graph backend, the graph is deleted before import +- Icon button tooltips do not pop up out of control +- Edge markers are displayed in workflow editor +- If a project is imported to the knowledge graph backend, the graph is deleted before import **v25.2.0 of DataIntegration adds the following new features:** -- New operators and dataset for improved file handling in workflows: - - **Add project files** workflow operator - Add files to projects directly from workflows - - **Get project files** workflow operator - Retrieve and process project files within workflow executions - - **Binary file dataset** - Handle binary files (PDF, images, etc.) in data integration pipelines -- **Neo4j database configuration** - Added parameter to configure specific databases in Neo4j connections -- **Project variable autocompletion** - All template operators now support autocompletion for project variables -- **Camel case transform operator** - Convert text to camel case format for data standardization -- **Project page URL suffix configuration** - New config key `workbench.project.defaultUrlSuffix` to configure the project page view (defaults to `?itemType=workflow&page=1&limit=10`) -- **Path auto-completion** - Mapping and linking rule editors now feature intelligent path auto-completion like in value mapping forms +- New operators and dataset for improved file handling in workflows: + - **Add project files** workflow operator - Add files to projects directly from workflows + - **Get project files** workflow operator - Retrieve and process project files within workflow executions + - **Binary file dataset** - Handle binary files (PDF, images, etc.) in data integration pipelines +- **Neo4j database configuration** - Added parameter to configure specific databases in Neo4j connections +- **Project variable autocompletion** - All template operators now support autocompletion for project variables +- **Camel case transform operator** - Convert text to camel case format for data standardization +- **Project page URL suffix configuration** - New config key `workbench.project.defaultUrlSuffix` to configure the project page view (defaults to `?itemType=workflow&page=1&limit=10`) +- **Path auto-completion** - Mapping and linking rule editors now feature intelligent path auto-completion like in value mapping forms **v25.2.0 of DataIntegration introduces the following changes:** -- **Infrastructure updates:** - - Migrated to Java 21 for improved performance and latest language features - - Updated Docker base image to `eclipse-temurin:21-ubi9-minimal` -- **"Internal dataset (single graph)"** added to plugins to properly display reports using this dataset type -- **Configurable favicon** - Organizations can customize the application favicon -- **JSON dataset improvements:** - - New parameter to control automatic navigation into JSON arrays - - New `#arrayPath` path operator for explicit navigation into JSON arrays (available when automatic JSON array navigation is set to `false`) - - New `#uuid` path operator generates type 3 (name-based) UUIDs from JSON node string representations - - New `#arrayText` path operator for enhanced array value extraction +- **Infrastructure updates:** + - Migrated to Java 21 for improved performance and latest language features + - Updated Docker base image to `eclipse-temurin:21-ubi9-minimal` +- **"Internal dataset (single graph)"** added to plugins to properly display reports using this dataset type +- **Configurable favicon** - Organizations can customize the application favicon +- **JSON dataset improvements:** + - New parameter to control automatic navigation into JSON arrays + - New `#arrayPath` path operator for explicit navigation into JSON arrays (available when automatic JSON array navigation is set to `false`) + - New `#uuid` path operator generates type 3 (name-based) UUIDs from JSON node string representations + - New `#arrayText` path operator for enhanced array value extraction **v25.2.0 of DataIntegration ships the following fixes:** -- Fixed queries with ORDER BY clauses in SQL dataset -- Fixed create task dialog focus issues when opened via 'connect to newly created...' menu option -- Fixed errors in Office 365 dataset tests and adapted to Microsoft API changes -- Fixed display issues for workflow reports containing internal datasets -- Fixed drag-and-drop problems when adding operators to nested workflow editors -- Non-printable characters in CSV datasets are now preserved during read/write transformations -- XML datasets now return empty values for empty tags when string values are expected -- Project variable updates now properly use the triggering user's credentials +- Fixed queries with ORDER BY clauses in SQL dataset +- Fixed create task dialog focus issues when opened via 'connect to newly created...' menu option +- Fixed errors in Office 365 dataset tests and adapted to Microsoft API changes +- Fixed display issues for workflow reports containing internal datasets +- Fixed drag-and-drop problems when adding operators to nested workflow editors +- Non-printable characters in CSV datasets are now preserved during read/write transformations +- XML datasets now return empty values for empty tags when string values are expected +- Project variable updates now properly use the triggering user's credentials ## eccenca Explore v25.2.6 @@ -99,81 +99,81 @@ We are pleased to announce Explore v25.2.6, which brings significant enhancement **v25.2.6 of Explore ships the following fix:** -- Removal of sub dependency redis -- Update Spring Boot to 3.5.7 +- Removal of sub dependency redis +- Update Spring Boot to 3.5.7 **v25.2.6 of Explore ships the following fix:** -- Removal of sub dependency redis -- Update Spring Boot to 3.5.7 +- Removal of sub dependency redis +- Update Spring Boot to 3.5.7 **v25.2.5 of Explore ships the following fix:** -- Anonymous classes are not considered when resolving the `rdfs:subClassOf` based class hierarchy +- Anonymous classes are not considered when resolving the `rdfs:subClassOf` based class hierarchy **v25.2.4 of Explore ships the following fix:** -- Add missing property for setting lifetime of stale (facet) caches +- Add missing property for setting lifetime of stale (facet) caches **v25.2.3 of Explore ships the following fixes and additions:** -- Protect charts module with the action 'Explore-ChartsCatalog' -- Fix critical Jinjava vulnerability +- Protect charts module with the action 'Explore-ChartsCatalog' +- Fix critical Jinjava vulnerability **v25.2.2 of Explore ships the following fixes and additions:** -- Update the gui-elements version to solve the edges cut problem +- Update the gui-elements version to solve the edges cut problem **v25.2.1 of Explore ships the following fixes and additions:** -- Pagination for inline views of Dataset (used for preview in cmem build) -- Widgets for shaped resources without properties to display are shown -- Update/Replace in the explore graph list +- Pagination for inline views of Dataset (used for preview in cmem build) +- Widgets for shaped resources without properties to display are shown +- Update/Replace in the explore graph list **v25.2.0 of Explore adds the following new features:** -- **Enhanced SHACL Shape Management:** - - Create target classes for node shapes on-the-fly during shape definition - - Create properties for property shapes on-the-fly without leaving the shape editor - - Create classes for property shapes on-the-fly for better data modeling - - Support for defining properties with `domainIncludes` and `rangeIncludes` predicates (as defined in either `schema:`,`dcam:` or `gist:`) -- **Query Catalog Enhancements:** - - Graph selection support for Query Catalog, allowing multiple query catalog graphs and editing queries in arbitrary graphs - - Graph selection support for Charts visualization, allowing to store and edit chart visualization in arbitrary graphs -- **GraphDB 11.0.x Support** - Full compatibility with the latest GraphDB version -- **Unified Error Handling** - New RTKAction handler provides consistent error handling across the application +- **Enhanced SHACL Shape Management:** + - Create target classes for node shapes on-the-fly during shape definition + - Create properties for property shapes on-the-fly without leaving the shape editor + - Create classes for property shapes on-the-fly for better data modeling + - Support for defining properties with `domainIncludes` and `rangeIncludes` predicates (as defined in either `schema:`,`dcam:` or `gist:`) +- **Query Catalog Enhancements:** + - Graph selection support for Query Catalog, allowing multiple query catalog graphs and editing queries in arbitrary graphs + - Graph selection support for Charts visualization, allowing to store and edit chart visualization in arbitrary graphs +- **GraphDB 11.0.x Support** - Full compatibility with the latest GraphDB version +- **Unified Error Handling** - New RTKAction handler provides consistent error handling across the application **v25.2.0 of Explore introduces the following changes:** -- **Infrastructure Updates:** - - Upgraded to Spring Boot 3.5.x and Apache Jena 5.4 - - Migrated to Java 21 runtime for improved performance -- **Timetracker Module** - Complete rework of the Timetracker and reports module for better performance and usability -- **Knowledge Graph Editor (BKE) Improvements:** - - Updated to React Flow v12 for enhanced graph visualization - - Automatic canvas scrolling when dragging items beyond visible area - - Advanced multi-select functionality on canvas for bulk operations -- **SPARQL Query Endpoints** - Changed to use an explicit list of allowed content types for better security -- **Catalog Query Management** - Update and SELECT queries are now differentiated by `rdf:type` using `shui:SparqlQuery` or `shui:UpdateQuery` +- **Infrastructure Updates:** + - Upgraded to Spring Boot 3.5.x and Apache Jena 5.4 + - Migrated to Java 21 runtime for improved performance +- **Timetracker Module** - Complete rework of the Timetracker and reports module for better performance and usability +- **Knowledge Graph Editor (BKE) Improvements:** + - Updated to React Flow v12 for enhanced graph visualization + - Automatic canvas scrolling when dragging items beyond visible area + - Advanced multi-select functionality on canvas for bulk operations +- **SPARQL Query Endpoints** - Changed to use an explicit list of allowed content types for better security +- **Catalog Query Management** - Update and SELECT queries are now differentiated by `rdf:type` using `shui:SparqlQuery` or `shui:UpdateQuery` **v25.2.0 of Explore ships the following fixes:** -- **Query Catalog:** - - Fixed SPARQL Query editor behavior after "save as" operation - - Improved error handling in Query Catalog API -- **Knowledge Graph Editor (BKE):** - - Property shape descriptions now consistently display as tooltips - - Fixed selection issues with expanded nodes - - Fixed greyed-out entries in Initial Search & Explore Navigation Box - - Resolved highlight lag issues for better performance -- **General Fixes:** - - Added warning when "New graph from File" overwrites existing graphs - - Empty node shapes are now properly hidden - - Fixed SHACL Edit Validation Button stability issues - - Fixed SHACL MaxCount property behavior - - Resolved duplicate entries in ResourceManager table - - Fixed broken resource selection for domain and range when graphs contain complex classes - - Enabled empty GSP multipart file uploads +- **Query Catalog:** + - Fixed SPARQL Query editor behavior after "save as" operation + - Improved error handling in Query Catalog API +- **Knowledge Graph Editor (BKE):** + - Property shape descriptions now consistently display as tooltips + - Fixed selection issues with expanded nodes + - Fixed greyed-out entries in Initial Search & Explore Navigation Box + - Resolved highlight lag issues for better performance +- **General Fixes:** + - Added warning when "New graph from File" overwrites existing graphs + - Empty node shapes are now properly hidden + - Fixed SHACL Edit Validation Button stability issues + - Fixed SHACL MaxCount property behavior + - Resolved duplicate entries in ResourceManager table + - Fixed broken resource selection for domain and range when graphs contain complex classes + - Enabled empty GSP multipart file uploads ## eccenca Corporate Memory Control (cmemc) v25.4.0 @@ -186,36 +186,36 @@ We are excited to announce cmemc v25.4.0, which introduces new features, improve **v25.4.0 of cmemc adds the following new features:** -- `query` command group - - can be used with arbitrary query graphs now - - `query list` command - new `--catalog-graph` option to select query catalog - - `query execute` command - new `--catalog-graph` option to select query catalog - - `query open` command - new `--catalog-graph` option to select query catalog +- `query` command group + - can be used with arbitrary query graphs now + - `query list` command - new `--catalog-graph` option to select query catalog + - `query execute` command - new `--catalog-graph` option to select query catalog + - `query open` command - new `--catalog-graph` option to select query catalog **v25.3.0 of cmemc adds the following new features:** -- `dataset create` command - - support for binary file datasets - - suggest pdf, png, jpg, jpeg, gif and tiff files as binary file dataset - - shell completion of these files -- `workflow io` command - - support for binary file datasets - - accept `application/octet-stream` as mime type for input and output files - - shell completion of pdf, png, jpg, jpeg, gif and tiff files as input and output - - add support for markdown documents as text datasets +- `dataset create` command + - support for binary file datasets + - suggest pdf, png, jpg, jpeg, gif and tiff files as binary file dataset + - shell completion of these files +- `workflow io` command + - support for binary file datasets + - accept `application/octet-stream` as mime type for input and output files + - shell completion of pdf, png, jpg, jpeg, gif and tiff files as input and output + - add support for markdown documents as text datasets **v25.2.0 of cmemc adds the following new features:** -- `graph imports` command group - - `graph imports create` command - Add graph import to a graph - - `graph imports delete` command - Delete graph import from a graph - - `graph imports list` command - List accessible graph's imports -- `graph export` command - - `--include-import-statements` option to save a `*.imports` file preserving imports of a graph -- `graph import` command - - `--include-import-statements` option to read the `*.imports` files and add the preserved imports to the store -- `graph delete` command - - `--include-import-statements` option to delete imports from other graphs to the deleted graph +- `graph imports` command group + - `graph imports create` command - Add graph import to a graph + - `graph imports delete` command - Delete graph import from a graph + - `graph imports list` command - List accessible graph's imports +- `graph export` command + - `--include-import-statements` option to save a `*.imports` file preserving imports of a graph +- `graph import` command + - `--include-import-statements` option to read the `*.imports` files and add the preserved imports to the store +- `graph delete` command + - `--include-import-statements` option to delete imports from other graphs to the deleted graph ## Migration Notes @@ -234,20 +234,20 @@ We are excited to announce cmemc v25.4.0, which introduces new features, improve ### eccenca DataIntegration -- The following plugins have been deprecated and will be removed in a future release: - - Old Python plugins depending on Jython (Python 2.x) - - Spark scripting plugins - - Spark virtual dataset - - Legacy REST operator -- To check if your instance uses any deprecated plugins, use the endpoint: `GET {DataIntegrationURL}/api/core/usages/deprecatedPlugins` +- The following plugins have been deprecated and will be removed in a future release: + - Old Python plugins depending on Jython (Python 2.x) + - Spark scripting plugins + - Spark virtual dataset + - Legacy REST operator +- To check if your instance uses any deprecated plugins, use the endpoint: `GET {DataIntegrationURL}/api/core/usages/deprecatedPlugins` ### eccenca Explore -- **Query Catalog Query Type Changes** - Catalog managed queries no longer persist `shui:queryType`. Update and SELECT queries are now differentiated by `rdf:type`: - - SELECT queries use `shui:SparqlQuery` - - UPDATE queries use `shui:UpdateQuery` +- **Query Catalog Query Type Changes** - Catalog managed queries no longer persist `shui:queryType`. Update and SELECT queries are now differentiated by `rdf:type`: + - SELECT queries use `shui:SparqlQuery` + - UPDATE queries use `shui:UpdateQuery` ### eccenca Corporate Memory Control (cmemc) -- With the introduction of the `graph imports` command group, the `graph tree` command is now deprecated. - - use `graph imports tree` instead +- With the introduction of the `graph imports` command group, the `graph tree` command is now deprecated. + - use `graph imports tree` instead diff --git a/docs/release-notes/corporate-memory-25-3/index.md b/docs/release-notes/corporate-memory-25-3/index.md index ef751eab..4b15ff98 100644 --- a/docs/release-notes/corporate-memory-25-3/index.md +++ b/docs/release-notes/corporate-memory-25-3/index.md @@ -15,16 +15,19 @@ Corporate Memory 25.3 is the third major release in 2025. The highlights of this release are: -- Build: **Mapping Creator** - - Experience a **next-generation, GenAI-driven visual mapping tool** that redefines how you build and manage mapping rules—with unparalleled clarity, speed, and simplicity. +- Build: **Mapping Creator** + - Experience a **next-generation, GenAI-driven visual mapping tool** that redefines how you build and manage mapping rules—with unparalleled clarity, speed, and simplicity. + +- Explore: **Companion Chat-Based Data Interaction** + - Introducing an **LLM-powered conversational interface** that lets you interact directly with your data. -- Explore: **Companion Chat-Based Data Interaction** - - Introducing an **LLM-powered conversational interface** that lets you interact directly with your data. Ask questions about your graphs, explore insights using query catalogs or autogenerated queries, and access Corporate Memory resources and modules—all within chat. + This feature takes data accessibility and interaction to an entirely new level. -- Explore: **Graph Insights** - - Visually explore your graphs contents and relationships in an **aggregated representation** that scales easily to graphs with millions of nodes. +- Explore: **Graph Insights** + - Visually explore your graphs contents and relationships in an **aggregated representation** that scales easily to graphs with millions of nodes. + Offering a **compact and flexible overview** aggregating objects and relations and shows details on demand. - diff --git a/docs/testing.md b/docs/testing.md index f477b166..debc767f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1,4 +1,5 @@ + # Testing Current issues: @@ -17,9 +18,9 @@ Current issues: ### Example from Geometric mean: 1 -* Weights: [1, 2, 1] -* Input values: [0.0, 0.0, 0.0] -* Returns: `0.0` +- Weights: [1, 2, 1] +- Input values: [0.0, 0.0, 0.0] +- Returns: `0.0` !!! example "Example from Geometric mean: 1" @@ -52,11 +53,11 @@ Current issues: ### Example from Handle missing values: Outputs the default score, if no input score is provided -* Parameters - * *defaultValue*: `1.0` +- Parameters + - *defaultValue*: `1.0` -* Input values: [(none)] -* Returns: `1.0` +- Input values: [(none)] +- Returns: `1.0` !!! example "Outputs the default score, if no input score is provided" @@ -80,11 +81,11 @@ Current issues: ### Example from Date: Returns 0 if both dates are equal -* Input values: - * Source: `[2003-03-01]` - * Target: `[2003-03-01]` +- Input values: + - Source: `[2003-03-01]` + - Target: `[2003-03-01]` -* Returns: → `0.0` +- Returns: → `0.0` !!! example "Returns 0 if both dates are equal" @@ -105,25 +106,25 @@ Current issues: ### Example from Camel Case: A sentence with several words is converted to a single word written in UpperCamelCase -* Parameters - * *isDromedary*: `false` +- Parameters + - *isDromedary*: `false` -* Input values: - 1. `[hello world]` +- Input values: + 1. `[hello world]` -* Returns: +- Returns: → `[HelloWorld]` ### Example from Coalesce: 5 -* Input values: +- Input values: 1. `[]` 2. `[first A, first B]` 3. `[second]` -* Returns: +- Returns: → `[first A, first B]` @@ -150,32 +151,34 @@ Current issues: ### Example of Concatenate: 4 -* Parameters - * *glue*: `-` +- Parameters + - *glue*: `-` -* Input values: - 1. `[First]` - 2. `[Last]` +- Input values: + 1. `[First]` + 2. `[Last]` -* Returns: +- Returns: → `[First-Last]` ### Example of Concatenate multiple values: 6 -* Parameters - * *glue*: `\n\t\\` +- Parameters + - *glue*: `\n\t\\` -* Input values: - 1. `[a - \b, c]` +- Input values: + 1. `[a + \b, c]` -* Returns: +- Returns: → `[a - \b - \c]` + + \b + + \c]` !!! example "Example 5" @@ -200,4 +203,3 @@ Current issues: \b \c] ``` - diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index fa07c3eb..86a02e21 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -14,4 +14,3 @@ hide: - diff --git a/poetry.lock b/poetry.lock index 5288358b..4f4bfaf3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -6,7 +6,6 @@ version = "2.6.1" description = "Happy Eyeballs for asyncio" optional = false python-versions = ">=3.9" -groups = ["dev"] files = [ {file = "aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8"}, {file = "aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558"}, @@ -18,7 +17,6 @@ version = "3.12.13" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.9" -groups = ["dev"] files = [ {file = "aiohttp-3.12.13-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5421af8f22a98f640261ee48aae3a37f0c41371e99412d55eaf2f8a46d5dad29"}, {file = "aiohttp-3.12.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0fcda86f6cb318ba36ed8f1396a6a4a3fd8f856f84d426584392083d10da4de0"}, @@ -126,7 +124,6 @@ version = "1.3.2" description = "aiosignal: a list of registered asynchronous callbacks" optional = false python-versions = ">=3.9" -groups = ["dev"] files = [ {file = "aiosignal-1.3.2-py2.py3-none-any.whl", hash = "sha256:45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5"}, {file = "aiosignal-1.3.2.tar.gz", hash = "sha256:a8c255c66fafb1e499c9351d0bf32ff2d8a0321595ebac3b93713656d2436f54"}, @@ -141,7 +138,6 @@ version = "0.7.0" description = "Reusable constraint types to use with typing.Annotated" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, @@ -153,7 +149,6 @@ version = "25.3.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.8" -groups = ["dev"] files = [ {file = "attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3"}, {file = "attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b"}, @@ -173,7 +168,6 @@ version = "2.17.0" description = "Internationalization utilities" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2"}, {file = "babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d"}, @@ -188,7 +182,6 @@ version = "5.9" description = "A wrapper around re and regex that adds additional back references." optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "backrefs-5.9-py310-none-any.whl", hash = "sha256:db8e8ba0e9de81fcd635f440deab5ae5f2591b54ac1ebe0550a2ca063488cd9f"}, {file = "backrefs-5.9-py311-none-any.whl", hash = "sha256:6907635edebbe9b2dc3de3a2befff44d74f30a4562adbb8b36f21252ea19c5cf"}, @@ -208,7 +201,6 @@ version = "4.13.4" description = "Screen-scraping library" optional = false python-versions = ">=3.7.0" -groups = ["main"] files = [ {file = "beautifulsoup4-4.13.4-py3-none-any.whl", hash = "sha256:9bbbb14bfde9d79f38b8cd5f8c7c85f4b8f2523190ebed90e950a8dea4cb1c4b"}, {file = "beautifulsoup4-4.13.4.tar.gz", hash = "sha256:dbb3c4e1ceae6aefebdaf2423247260cd062430a410e38c66f2baa50a8437195"}, @@ -231,7 +223,6 @@ version = "2.6" description = "Bash style brace expander." optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "bracex-2.6-py3-none-any.whl", hash = "sha256:0b0049264e7340b3ec782b5cb99beb325f36c3782a32e36e876452fd49a09952"}, {file = "bracex-2.6.tar.gz", hash = "sha256:98f1347cd77e22ee8d967a30ad4e310b233f7754dbf31ff3fceb76145ba47dc7"}, @@ -243,7 +234,6 @@ version = "1.7.1" description = "cffi-based cairo bindings for Python" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "cairocffi-1.7.1-py3-none-any.whl", hash = "sha256:9803a0e11f6c962f3b0ae2ec8ba6ae45e957a146a004697a1ac1bbf16b073b3f"}, {file = "cairocffi-1.7.1.tar.gz", hash = "sha256:2e48ee864884ec4a3a34bfa8c9ab9999f688286eb714a15a43ec9d068c36557b"}, @@ -263,7 +253,6 @@ version = "2.8.2" description = "A Simple SVG Converter based on Cairo" optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "cairosvg-2.8.2-py3-none-any.whl", hash = "sha256:eab46dad4674f33267a671dce39b64be245911c901c70d65d2b7b0821e852bf5"}, {file = "cairosvg-2.8.2.tar.gz", hash = "sha256:07cbf4e86317b27a92318a4cac2a4bb37a5e9c1b8a27355d06874b22f85bef9f"}, @@ -286,7 +275,6 @@ version = "2025.6.15" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" -groups = ["main"] files = [ {file = "certifi-2025.6.15-py3-none-any.whl", hash = "sha256:2e0c7ce7cb5d8f8634ca55d2ba7e6ec2689a2fd6537d8dec1296a477a4910057"}, {file = "certifi-2025.6.15.tar.gz", hash = "sha256:d747aa5a8b9bbbb1bb8c22bb13e22bd1f18e9796defa16bab421f7f7a317323b"}, @@ -298,7 +286,6 @@ version = "1.17.1" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, @@ -378,7 +365,6 @@ version = "3.4.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7" -groups = ["main"] files = [ {file = "charset_normalizer-3.4.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941"}, {file = "charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd"}, @@ -480,7 +466,6 @@ version = "8.2.1" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.10" -groups = ["main"] files = [ {file = "click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b"}, {file = "click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202"}, @@ -495,7 +480,6 @@ version = "25.3.0" description = "API for eccenca Corporate Memory" optional = false python-versions = "<4.0,>=3.9" -groups = ["main"] files = [ {file = "cmem_cmempy-25.3.0-py3-none-any.whl", hash = "sha256:75f9c6900661b5573615b43086897eb4b5fccdb1ec953fa9e20cdaecaeea75c2"}, {file = "cmem_cmempy-25.3.0.tar.gz", hash = "sha256:ccef1410bde7e248d4b89b37366e7c386c8a1558190a07090f0d3c11e3b16ff4"}, @@ -514,7 +498,6 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -526,7 +509,6 @@ version = "0.8.0" description = "CSS selectors for Python ElementTree" optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "cssselect2-0.8.0-py3-none-any.whl", hash = "sha256:46fc70ebc41ced7a32cd42d58b1884d72ade23d21e5a4eaaf022401c13f0e76e"}, {file = "cssselect2-0.8.0.tar.gz", hash = "sha256:7674ffb954a3b46162392aee2a3a0aedb2e14ecf99fcc28644900f4e6e3e9d3a"}, @@ -546,7 +528,6 @@ version = "0.7.1" description = "XML bomb protection for Python stdlib modules" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -groups = ["main"] files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -558,7 +539,6 @@ version = "1.7.0" description = "A list-like structure which implements collections.abc.MutableSequence" optional = false python-versions = ">=3.9" -groups = ["dev"] files = [ {file = "frozenlist-1.7.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cc4df77d638aa2ed703b878dd093725b72a824c3c546c076e8fdf276f78ee84a"}, {file = "frozenlist-1.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:716a9973a2cc963160394f701964fe25012600f3d311f60c790400b00e568b61"}, @@ -672,7 +652,6 @@ version = "2.1.0" description = "Copy your docs directly to the gh-pages branch." optional = false python-versions = "*" -groups = ["main"] files = [ {file = "ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343"}, {file = "ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619"}, @@ -690,7 +669,6 @@ version = "4.0.12" description = "Git Object Database" optional = false python-versions = ">=3.7" -groups = ["main"] files = [ {file = "gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf"}, {file = "gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571"}, @@ -705,7 +683,6 @@ version = "3.1.44" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" -groups = ["main"] files = [ {file = "GitPython-3.1.44-py3-none-any.whl", hash = "sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110"}, {file = "gitpython-3.1.44.tar.gz", hash = "sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269"}, @@ -724,7 +701,6 @@ version = "3.1.0" description = "Hjson, a user interface for JSON." optional = false python-versions = "*" -groups = ["main"] files = [ {file = "hjson-3.1.0-py3-none-any.whl", hash = "sha256:65713cdcf13214fb554eb8b4ef803419733f4f5e551047c9b711098ab7186b89"}, {file = "hjson-3.1.0.tar.gz", hash = "sha256:55af475a27cf83a7969c808399d7bccdec8fb836a07ddbd574587593b9cdcf75"}, @@ -736,7 +712,6 @@ version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" -groups = ["main", "dev"] files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, @@ -751,7 +726,6 @@ version = "8.7.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd"}, {file = "importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000"}, @@ -775,7 +749,6 @@ version = "6.5.2" description = "Read resources from Python packages" optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "importlib_resources-6.5.2-py3-none-any.whl", hash = "sha256:789cfdc3ed28c78b67a06acb8126751ced69a3d5f79c095a98298cd8a760ccec"}, {file = "importlib_resources-6.5.2.tar.gz", hash = "sha256:185f87adef5bcc288449d98fb4fba07cea78bc036455dd44c5fc4a2fe78fed2c"}, @@ -795,7 +768,6 @@ version = "3.1.6" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" -groups = ["main"] files = [ {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, @@ -813,7 +785,6 @@ version = "1.4.0" description = "Check links for Markdown-based site" optional = false python-versions = ">=3.7" -groups = ["dev"] files = [ {file = "linkcheckmd-1.4.0.tar.gz", hash = "sha256:3a539c9a4e11697fc7fcc269d379accf93c8cccbf971f3cea0bae40912d9f609"}, ] @@ -832,7 +803,6 @@ version = "3.8.2" description = "Python implementation of John Gruber's Markdown." optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "markdown-3.8.2-py3-none-any.whl", hash = "sha256:5c83764dbd4e00bdd94d85a19b8d55ccca20fe35b2e678a1422b380324dd5f24"}, {file = "markdown-3.8.2.tar.gz", hash = "sha256:247b9a70dd12e27f67431ce62523e675b866d254f900c4fe75ce3dda62237c45"}, @@ -848,7 +818,6 @@ version = "3.0.2" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"}, @@ -919,7 +888,6 @@ version = "1.3.4" description = "A deep merge function for 🐍." optional = false python-versions = ">=3.6" -groups = ["main"] files = [ {file = "mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307"}, {file = "mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8"}, @@ -931,7 +899,6 @@ version = "2.1.3" description = "Manage multiple versions of your MkDocs-powered documentation" optional = false python-versions = "*" -groups = ["main"] files = [ {file = "mike-2.1.3-py3-none-any.whl", hash = "sha256:d90c64077e84f06272437b464735130d380703a76a5738b152932884c60c062a"}, {file = "mike-2.1.3.tar.gz", hash = "sha256:abd79b8ea483fb0275b7972825d3082e5ae67a41820f8d8a0dc7a3f49944e810"}, @@ -957,7 +924,6 @@ version = "1.6.1" description = "Project documentation with Markdown." optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e"}, {file = "mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2"}, @@ -988,7 +954,6 @@ version = "0.7.1" description = "An MkDocs plugin" optional = false python-versions = ">=3.4" -groups = ["main"] files = [ {file = "mkdocs-autolinks-plugin-0.7.1.tar.gz", hash = "sha256:445ddb9b417b7795856c30801bb430773186c1daf210bdeecf8305f55a47d151"}, {file = "mkdocs_autolinks_plugin-0.7.1-py3-none-any.whl", hash = "sha256:5c6c17f6649b68e79a9ef0b2648d59f3072e18002b90ee1586a64c505f11ab12"}, @@ -1003,7 +968,6 @@ version = "2.10.1" description = "An MkDocs plugin that simplifies configuring page titles and their order" optional = false python-versions = ">=3.8.1" -groups = ["main"] files = [ {file = "mkdocs_awesome_pages_plugin-2.10.1-py3-none-any.whl", hash = "sha256:c6939dbea37383fc3cf8c0a4e892144ec3d2f8a585e16fdc966b34e7c97042a7"}, {file = "mkdocs_awesome_pages_plugin-2.10.1.tar.gz", hash = "sha256:cda2cb88c937ada81a4785225f20ef77ce532762f4500120b67a1433c1cdbb2f"}, @@ -1020,7 +984,6 @@ version = "0.2.0" description = "MkDocs extension that lists all dependencies according to a mkdocs.yml file" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "mkdocs_get_deps-0.2.0-py3-none-any.whl", hash = "sha256:2bf11d0b133e77a0dd036abeeb06dec8775e46efa526dc70667d8863eefc6134"}, {file = "mkdocs_get_deps-0.2.0.tar.gz", hash = "sha256:162b3d129c7fad9b19abfdcb9c1458a651628e4b1dea628ac68790fb3061c60c"}, @@ -1037,7 +1000,6 @@ version = "1.4.7" description = "Mkdocs plugin that enables displaying the localized date of the last git modification of a markdown file." optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "mkdocs_git_revision_date_localized_plugin-1.4.7-py3-none-any.whl", hash = "sha256:056c0a90242409148f1dc94d5c9d2c25b5b8ddd8de45489fa38f7fa7ccad2bc4"}, {file = "mkdocs_git_revision_date_localized_plugin-1.4.7.tar.gz", hash = "sha256:10a49eff1e1c3cb766e054b9d8360c904ce4fe8c33ac3f6cc083ac6459c91953"}, @@ -1055,7 +1017,6 @@ version = "0.4.0" description = "MkDocs plugin supports image lightbox with GLightbox." optional = false python-versions = "*" -groups = ["main"] files = [ {file = "mkdocs-glightbox-0.4.0.tar.gz", hash = "sha256:392b34207bf95991071a16d5f8916d1d2f2cd5d5bb59ae2997485ccd778c70d9"}, {file = "mkdocs_glightbox-0.4.0-py3-none-any.whl", hash = "sha256:e0107beee75d3eb7380ac06ea2d6eac94c999eaa49f8c3cbab0e7be2ac006ccf"}, @@ -1067,7 +1028,6 @@ version = "1.3.7" description = "Unleash the power of MkDocs with macros and variables" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "mkdocs_macros_plugin-1.3.7-py3-none-any.whl", hash = "sha256:02432033a5b77fb247d6ec7924e72fc4ceec264165b1644ab8d0dc159c22ce59"}, {file = "mkdocs_macros_plugin-1.3.7.tar.gz", hash = "sha256:17c7fd1a49b94defcdb502fd453d17a1e730f8836523379d21292eb2be4cb523"}, @@ -1093,7 +1053,6 @@ version = "9.6.14+insiders.4.53.16" description = "Documentation that simply works" optional = false python-versions = ">=3.8" -groups = ["main"] files = [] develop = false @@ -1118,7 +1077,7 @@ recommended = ["mkdocs-minify-plugin (>=0.7,<1.0)", "mkdocs-redirects (>=1.2,<2. [package.source] type = "git" -url = "ssh://git@github.com/eccenca/mkdocs-material-insiders.git" +url = "git@github.com:eccenca/mkdocs-material-insiders.git" reference = "9.6.14-insiders-4.53.16" resolved_reference = "ce2cca8c5240ae520e09a67954de09949bd04efe" @@ -1128,7 +1087,6 @@ version = "1.3.1" description = "Extension pack for Python Markdown and MkDocs Material." optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31"}, {file = "mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443"}, @@ -1140,7 +1098,6 @@ version = "1.2.2" description = "A MkDocs plugin for dynamic page redirects to prevent broken links" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "mkdocs_redirects-1.2.2-py3-none-any.whl", hash = "sha256:7dbfa5647b79a3589da4401403d69494bd1f4ad03b9c15136720367e1f340ed5"}, {file = "mkdocs_redirects-1.2.2.tar.gz", hash = "sha256:3094981b42ffab29313c2c1b8ac3969861109f58b2dd58c45fc81cd44bfa0095"}, @@ -1155,7 +1112,6 @@ version = "0.7.1" description = "A MkDocs plugin supports for add Swagger UI in page." optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "mkdocs_swagger_ui_tag-0.7.1-py3-none-any.whl", hash = "sha256:e4a1019c96ef333ec4dab0ef7d80068a345c7526a87fe8718f18852ee5ad34a5"}, {file = "mkdocs_swagger_ui_tag-0.7.1.tar.gz", hash = "sha256:aed3c5f15297d74241f38cfba4763a5789bf10a410e005014763c66e79576b65"}, @@ -1170,7 +1126,6 @@ version = "6.6.0" description = "multidict implementation" optional = false python-versions = ">=3.9" -groups = ["dev"] files = [ {file = "multidict-6.6.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d7913e6d0953b6d65c74290da65bc33d60d32a48bbe0bf2398ea1c5a2626e0b2"}, {file = "multidict-6.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8552e89a546408d3f78f1efd1c48e46077b68e59b6d5607498dd0a44df60b87c"}, @@ -1278,7 +1233,6 @@ version = "8.4.0" description = "Simple yet flexible natural sorting in Python." optional = false python-versions = ">=3.7" -groups = ["main"] files = [ {file = "natsort-8.4.0-py3-none-any.whl", hash = "sha256:4732914fb471f56b5cce04d7bae6f164a592c7712e1c85f9ef585e197299521c"}, {file = "natsort-8.4.0.tar.gz", hash = "sha256:45312c4a0e5507593da193dedd04abb1469253b601ecaf63445ad80f0a1ea581"}, @@ -1294,7 +1248,6 @@ version = "25.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, @@ -1306,7 +1259,6 @@ version = "0.5.7" description = "Divides large result sets into pages for easier browsing" optional = false python-versions = "*" -groups = ["main"] files = [ {file = "paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591"}, {file = "paginate-0.5.7.tar.gz", hash = "sha256:22bd083ab41e1a8b4f3690544afb2c60c25e5c9a63a30fa2f483f6c60c8e5945"}, @@ -1322,7 +1274,6 @@ version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, @@ -1334,7 +1285,6 @@ version = "11.2.1" description = "Python Imaging Library (Fork)" optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "pillow-11.2.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:d57a75d53922fc20c165016a20d9c44f73305e67c351bbc60d1adaf662e74047"}, {file = "pillow-11.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:127bf6ac4a5b58b3d32fc8289656f77f80567d65660bc46f72c0d77e6600cc95"}, @@ -1434,7 +1384,6 @@ version = "4.3.8" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4"}, {file = "platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc"}, @@ -1451,7 +1400,6 @@ version = "0.3.2" description = "Accelerated property cache" optional = false python-versions = ">=3.9" -groups = ["dev"] files = [ {file = "propcache-0.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:22d9962a358aedbb7a2e36187ff273adeaab9743373a272976d2e348d08c7770"}, {file = "propcache-0.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0d0fda578d1dc3f77b6b5a5dce3b9ad69a8250a891760a548df850a5e8da87f3"}, @@ -1559,7 +1507,6 @@ version = "2.22" description = "C parser in Python" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, @@ -1571,7 +1518,6 @@ version = "2.11.7" description = "Data validation using Python type hints" optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "pydantic-2.11.7-py3-none-any.whl", hash = "sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b"}, {file = "pydantic-2.11.7.tar.gz", hash = "sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db"}, @@ -1593,7 +1539,6 @@ version = "2.33.2" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "pydantic_core-2.33.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8"}, {file = "pydantic_core-2.33.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d"}, @@ -1705,7 +1650,6 @@ version = "2.19.2" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, @@ -1720,7 +1664,6 @@ version = "10.16" description = "Extension pack for Python Markdown." optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "pymdown_extensions-10.16-py3-none-any.whl", hash = "sha256:f5dd064a4db588cb2d95229fc4ee63a1b16cc8b4d0e6145c0899ed8723da1df2"}, {file = "pymdown_extensions-10.16.tar.gz", hash = "sha256:71dac4fca63fabeffd3eb9038b756161a33ec6e8d230853d3cecf562155ab3de"}, @@ -1739,7 +1682,6 @@ version = "3.2.3" description = "pyparsing module - Classes and methods to define and execute parsing grammars" optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "pyparsing-3.2.3-py3-none-any.whl", hash = "sha256:a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf"}, {file = "pyparsing-3.2.3.tar.gz", hash = "sha256:b9c13f1ab8b3b542f72e28f634bad4de758ab3ce4546e4301970ad6fa77c38be"}, @@ -1754,7 +1696,6 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -1769,7 +1710,6 @@ version = "2025.2" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" -groups = ["main"] files = [ {file = "pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00"}, {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"}, @@ -1781,7 +1721,6 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -1844,7 +1783,6 @@ version = "1.1" description = "A custom YAML tag for referencing environment variables in YAML files." optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04"}, {file = "pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff"}, @@ -1859,7 +1797,6 @@ version = "7.1.4" description = "RDFLib is a Python library for working with RDF, a simple yet powerful language for representing information." optional = false python-versions = "<4.0.0,>=3.8.1" -groups = ["main"] files = [ {file = "rdflib-7.1.4-py3-none-any.whl", hash = "sha256:72f4adb1990fa5241abd22ddaf36d7cafa5d91d9ff2ba13f3086d339b213d997"}, {file = "rdflib-7.1.4.tar.gz", hash = "sha256:fed46e24f26a788e2ab8e445f7077f00edcf95abb73bcef4b86cefa8b62dd174"}, @@ -1881,7 +1818,6 @@ version = "2.32.4" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c"}, {file = "requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422"}, @@ -1903,7 +1839,6 @@ version = "1.0.0" description = "A utility belt for advanced users of python-requests" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -groups = ["main"] files = [ {file = "requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6"}, {file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"}, @@ -1912,13 +1847,29 @@ files = [ [package.dependencies] requests = ">=2.0.1,<3.0.0" +[[package]] +name = "rumdl" +version = "0.0.194" +description = "A fast Markdown linter written in Rust" +optional = false +python-versions = ">=3.7" +files = [ + {file = "rumdl-0.0.194-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:dbfdf073349795d06ef9b9b1b506495ea848a0074c1645a10056fb4ec633eaea"}, + {file = "rumdl-0.0.194-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b07b2c1e8ed5b2dcba5772259d3bee0a47711341d11055e289abcb9ca4348c0b"}, + {file = "rumdl-0.0.194-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:111af9c12f4a8d7a0a69152557962fb4d0467d1ed2444588b70ad299efc5a9ad"}, + {file = "rumdl-0.0.194-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31bec50ff51ae80909de7c8ceb2355a19c7f9f046f29624337e6a21dddc06afe"}, + {file = "rumdl-0.0.194-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c14f2d0468b4b41e0900d0e269deafa69d0cea0cda7eace886987e0f21cdd53f"}, + {file = "rumdl-0.0.194-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:459f97f8527cbc8620cfbe54d239cb480622f205f26e18fb6fa00ac204d1a93b"}, + {file = "rumdl-0.0.194-py3-none-win_amd64.whl", hash = "sha256:5b18d0d2f753030ae23374b1419b4872ca583257d12c03367c3000d0d1a5ad6f"}, + {file = "rumdl-0.0.194.tar.gz", hash = "sha256:be57d294e95c3dbf112cb408204bb0fd9fc5fe6ffb7df4b93e4579b565fcc51e"}, +] + [[package]] name = "six" version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -1930,7 +1881,6 @@ version = "5.0.2" description = "A pure Python implementation of a sliding window memory map manager" optional = false python-versions = ">=3.7" -groups = ["main"] files = [ {file = "smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e"}, {file = "smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5"}, @@ -1942,7 +1892,6 @@ version = "2.7" description = "A modern CSS selector implementation for Beautiful Soup." optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "soupsieve-2.7-py3-none-any.whl", hash = "sha256:6e60cc5c1ffaf1cebcc12e8188320b72071e922c2e897f737cadce79ad5d30c4"}, {file = "soupsieve-2.7.tar.gz", hash = "sha256:ad282f9b6926286d2ead4750552c8a6142bc4c783fd66b0293547c8fe6ae126a"}, @@ -1954,7 +1903,6 @@ version = "0.5.3" description = "file: README.md" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "super_collections-0.5.3-py3-none-any.whl", hash = "sha256:907d35b25dc4070910e8254bf2f5c928348af1cf8a1f1e8259e06c666e902cff"}, {file = "super_collections-0.5.3.tar.gz", hash = "sha256:94c1ec96c0a0d5e8e7d389ed8cde6882ac246940507c5e6b86e91945c2968d46"}, @@ -1972,7 +1920,6 @@ version = "3.1.0" description = "ANSI color formatting for output in terminal" optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "termcolor-3.1.0-py3-none-any.whl", hash = "sha256:591dd26b5c2ce03b9e43f391264626557873ce1d379019786f99b0c2bee140aa"}, {file = "termcolor-3.1.0.tar.gz", hash = "sha256:6a6dd7fbee581909eeec6a756cff1d7f7c376063b14e4a298dc4980309e55970"}, @@ -1987,7 +1934,6 @@ version = "1.4.0" description = "A tiny CSS parser" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "tinycss2-1.4.0-py3-none-any.whl", hash = "sha256:3a49cf47b7675da0b15d0c6e1df8df4ebd96e9394bb905a5775adb0d884c5289"}, {file = "tinycss2-1.4.0.tar.gz", hash = "sha256:10c0972f6fc0fbee87c3edb76549357415e94548c1ae10ebccdea16fb404a9b7"}, @@ -2006,7 +1952,6 @@ version = "4.14.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "typing_extensions-4.14.0-py3-none-any.whl", hash = "sha256:a1514509136dd0b477638fc68d6a91497af5076466ad0fa6c338e44e359944af"}, {file = "typing_extensions-4.14.0.tar.gz", hash = "sha256:8676b788e32f02ab42d9e7c61324048ae4c6d844a399eebace3d4979d75ceef4"}, @@ -2018,7 +1963,6 @@ version = "0.4.1" description = "Runtime typing introspection tools" optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51"}, {file = "typing_inspection-0.4.1.tar.gz", hash = "sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28"}, @@ -2033,7 +1977,6 @@ version = "2.5.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, @@ -2051,7 +1994,6 @@ version = "0.1.0" description = "Flexible version handling" optional = false python-versions = "*" -groups = ["main"] files = [ {file = "verspec-0.1.0-py3-none-any.whl", hash = "sha256:741877d5633cc9464c45a469ae2a31e801e6dbbaa85b9675d481cda100f11c31"}, {file = "verspec-0.1.0.tar.gz", hash = "sha256:c4504ca697b2056cdb4bfa7121461f5a0e81809255b41c03dda4ba823637c01e"}, @@ -2066,7 +2008,6 @@ version = "6.0.0" description = "Filesystem events monitoring" optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d1cdb490583ebd691c012b3d6dae011000fe42edb7a82ece80965b42abd61f26"}, {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc64ab3bdb6a04d69d4023b29422170b74681784ffb9463ed4870cf2f3e66112"}, @@ -2109,7 +2050,6 @@ version = "10.1" description = "Wildcard/glob file name matcher." optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "wcmatch-10.1-py3-none-any.whl", hash = "sha256:5848ace7dbb0476e5e55ab63c6bbd529745089343427caa5537f230cc01beb8a"}, {file = "wcmatch-10.1.tar.gz", hash = "sha256:f11f94208c8c8484a16f4f48638a85d771d9513f4ab3f37595978801cb9465af"}, @@ -2124,7 +2064,6 @@ version = "0.5.1" description = "Character encoding aliases for legacy web content" optional = false python-versions = "*" -groups = ["main"] files = [ {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, @@ -2136,7 +2075,6 @@ version = "1.20.1" description = "Yet another URL library" optional = false python-versions = ">=3.9" -groups = ["dev"] files = [ {file = "yarl-1.20.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6032e6da6abd41e4acda34d75a816012717000fa6839f37124a47fcefc49bec4"}, {file = "yarl-1.20.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2c7b34d804b8cf9b214f05015c4fee2ebe7ed05cf581e7192c06555c71f4446a"}, @@ -2255,7 +2193,6 @@ version = "3.23.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, @@ -2270,6 +2207,6 @@ test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_it type = ["pytest-mypy"] [metadata] -lock-version = "2.1" +lock-version = "2.0" python-versions = "^3.11" -content-hash = "a0e36e00724c0eadf029aa4d5c5358bf2942ef4cc6eb162dcb36e95a8aa7907d" +content-hash = "8dca9021c750c8ab63ab240ae05ccde92fb1998bcd1659ce19074d31610aefc9" diff --git a/pyproject.toml b/pyproject.toml index 644097b6..ff86d0d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ jinja2 = "^3.1.6" [tool.poetry.group.dev.dependencies] linkcheckmd = "^1.4.0" +rumdl = "^0.0.194" [build-system] requires = ["poetry-core"]