documents

UCloud Developer Guide / Orchestration of Resources / Storage / Metadata / Metadata Documents

Metadata Documents

Metadata documents form the foundation of data management in UCloud.

Rationale

UCloud supports arbitrary of files. This feature is useful for general data management. It allows users to tag documents at a glance and search through them.

This feature consists of two parts:

Metadata templates (previous section): Templates specify the schema. You can think of this as a way of defining how your documents should look. We use them to generate user interfaces and visual representations of your documents.
Metadata documents (you are here): Documents fill out the values of a template. When you create a document you must attach it to a file also.

1. Examples

2. Remote Procedure Calls

3. Data Models

Example: Sensitivity Document

Frequency of use Common

Frequency of use	Common
Actors	An authenticated user (`user`)

Actors

An authenticated user (user)

Communication Flow: Kotlin


/* In this example, we will show how to create a metadata document and attach it to a file. */


/* We already have a metadata template in the catalog: */

FileMetadataTemplateNamespaces.retrieveLatest.call(
    FindByStringId(
        id = "15123", 
    ),
    user
).orThrow()

/*
FileMetadataTemplate(
    changeLog = "Initial version", 
    createdAt = 0, 
    description = "File sensitivity for files", 
    inheritable = true, 
    namespaceId = "sensitivity", 
    namespaceName = null, 
    namespaceType = FileMetadataTemplateNamespaceType.COLLABORATORS, 
    requireApproval = true, 
    schema = JsonObject(mapOf("type" to JsonLiteral(
        coerceToInlineType = null, 
        content = "object", 
        isString = true, 
    )),"title" to JsonLiteral(
        coerceToInlineType = null, 
        content = "UCloud File Sensitivity", 
        isString = true, 
    )),"required" to listOf(JsonLiteral(
        coerceToInlineType = null, 
        content = "sensitivity", 
        isString = true, 
    ))),"properties" to JsonObject(mapOf("sensitivity" to JsonObject(mapOf("enum" to listOf(JsonLiteral(
        coerceToInlineType = null, 
        content = "SENSITIVE", 
        isString = true, 
    ), JsonLiteral(
        coerceToInlineType = null, 
        content = "CONFIDENTIAL", 
        isString = true, 
    ), JsonLiteral(
        coerceToInlineType = null, 
        content = "PRIVATE", 
        isString = true, 
    ))),"type" to JsonLiteral(
        coerceToInlineType = null, 
        content = "string", 
        isString = true, 
    )),"title" to JsonLiteral(
        coerceToInlineType = null, 
        content = "File Sensitivity", 
        isString = true, 
    )),"enumNames" to listOf(JsonLiteral(
        coerceToInlineType = null, 
        content = "Sensitive", 
        isString = true, 
    ), JsonLiteral(
        coerceToInlineType = null, 
        content = "Confidential", 
        isString = true, 
    ), JsonLiteral(
        coerceToInlineType = null, 
        content = "Private", 
        isString = true, 
    ))),))),))),"dependencies" to JsonObject(mapOf())),)), 
    title = "Sensitivity", 
    uiSchema = JsonObject(mapOf("ui:order" to listOf(JsonLiteral(
        coerceToInlineType = null, 
        content = "sensitivity", 
        isString = true, 
    ))),)), 
    version = "1.0.0", 
)
*/

/* Using this, we can create a metadata document and attach it to our file */

FileMetadata.create.call(
    bulkRequestOf(FileMetadataAddRequestItem(
        fileId = "/51231/my/file", 
        metadata = FileMetadataDocument.Spec(
            changeLog = "New sensitivity", 
            document = JsonObject(mapOf("sensitivity" to JsonLiteral(
                coerceToInlineType = null, 
                content = "SENSITIVE", 
                isString = true, 
            )),)), 
            templateId = "15123", 
            version = "1.0.0", 
        ), 
    )),
    user
).orThrow()

/*
BulkResponse(
    responses = listOf(FindByStringId(
        id = "651233", 
    )), 
)
*/

/* This specific template requires approval from a workspace admin. We can do this by calling approve. */

FileMetadata.approve.call(
    bulkRequestOf(FindByStringId(
        id = "651233", 
    )),
    user
).orThrow()

/*
Unit
*/

/* We can view the metadata by adding includeMetadata = true when requesting any file */

Files.retrieve.call(
    ResourceRetrieveRequest(
        flags = UFileIncludeFlags(
            allowUnsupportedInclude = null, 
            filterByFileExtension = null, 
            filterCreatedAfter = null, 
            filterCreatedBefore = null, 
            filterCreatedBy = null, 
            filterHiddenFiles = false, 
            filterIds = null, 
            filterProductCategory = null, 
            filterProductId = null, 
            filterProvider = null, 
            filterProviderIds = null, 
            hideProductCategory = null, 
            hideProductId = null, 
            hideProvider = null, 
            includeMetadata = true, 
            includeOthers = false, 
            includePermissions = null, 
            includeProduct = false, 
            includeSizes = null, 
            includeSupport = false, 
            includeTimestamps = null, 
            includeUnixInfo = null, 
            includeUpdates = false, 
            path = null, 
        ), 
        id = "51231", 
    ),
    user
).orThrow()

/*
UFile(
    createdAt = 1635151675465, 
    id = "/51231/my/file", 
    owner = ResourceOwner(
        createdBy = "user", 
        project = null, 
    ), 
    permissions = ResourcePermissions(
        myself = listOf(Permission.ADMIN), 
        others = emptyList(), 
    ), 
    specification = UFileSpecification(
        collection = "51231", 
        product = ProductReference(
            category = "example-ssd", 
            id = "example-ssd", 
            provider = "example", 
        ), 
    ), 
    status = UFileStatus(
        accessedAt = null, 
        icon = null, 
        metadata = FileMetadataHistory(
            metadata = mapOf("sensitivity" to listOf(FileMetadataDocument(
                createdAt = 1635151675465, 
                createdBy = "user", 
                id = "651233", 
                specification = FileMetadataDocument.Spec(
                    changeLog = "New sensitivity", 
                    document = JsonObject(mapOf("sensitivity" to JsonLiteral(
                        coerceToInlineType = null, 
                        content = "SENSITIVE", 
                        isString = true, 
                    )),)), 
                    templateId = "15123", 
                    version = "1.0.0", 
                ), 
                status = FileMetadataDocument.Status(
                    approval = FileMetadataDocument.ApprovalStatus.Approved(
                        approvedBy = "user", 
                    ), 
                ), 
            ))), 
            templates = mapOf("sensitivity" to FileMetadataTemplate(
                changeLog = "Initial version", 
                createdAt = 0, 
                description = "File sensitivity for files", 
                inheritable = true, 
                namespaceId = "sensitivity", 
                namespaceName = null, 
                namespaceType = FileMetadataTemplateNamespaceType.COLLABORATORS, 
                requireApproval = true, 
                schema = JsonObject(mapOf("type" to JsonLiteral(
                    coerceToInlineType = null, 
                    content = "object", 
                    isString = true, 
                )),"title" to JsonLiteral(
                    coerceToInlineType = null, 
                    content = "UCloud File Sensitivity", 
                    isString = true, 
                )),"required" to listOf(JsonLiteral(
                    coerceToInlineType = null, 
                    content = "sensitivity", 
                    isString = true, 
                ))),"properties" to JsonObject(mapOf("sensitivity" to JsonObject(mapOf("enum" to listOf(JsonLiteral(
                    coerceToInlineType = null, 
                    content = "SENSITIVE", 
                    isString = true, 
                ), JsonLiteral(
                    coerceToInlineType = null, 
                    content = "CONFIDENTIAL", 
                    isString = true, 
                ), JsonLiteral(
                    coerceToInlineType = null, 
                    content = "PRIVATE", 
                    isString = true, 
                ))),"type" to JsonLiteral(
                    coerceToInlineType = null, 
                    content = "string", 
                    isString = true, 
                )),"title" to JsonLiteral(
                    coerceToInlineType = null, 
                    content = "File Sensitivity", 
                    isString = true, 
                )),"enumNames" to listOf(JsonLiteral(
                    coerceToInlineType = null, 
                    content = "Sensitive", 
                    isString = true, 
                ), JsonLiteral(
                    coerceToInlineType = null, 
                    content = "Confidential", 
                    isString = true, 
                ), JsonLiteral(
                    coerceToInlineType = null, 
                    content = "Private", 
                    isString = true, 
                ))),))),))),"dependencies" to JsonObject(mapOf())),)), 
                title = "Sensitivity", 
                uiSchema = JsonObject(mapOf("ui:order" to listOf(JsonLiteral(
                    coerceToInlineType = null, 
                    content = "sensitivity", 
                    isString = true, 
                ))),)), 
                version = "1.0.0", 
            )), 
        ), 
        modifiedAt = null, 
        resolvedProduct = null, 
        resolvedSupport = null, 
        sizeInBytes = null, 
        sizeIncludingChildrenInBytes = null, 
        type = FileType.FILE, 
        unixGroup = null, 
        unixMode = null, 
        unixOwner = null, 
    ), 
    updates = emptyList(), 
    providerGeneratedId = "/51231/my/file", 
)
*/

Communication Flow: Curl

# ------------------------------------------------------------------------------------------------------
# $host is the UCloud instance to contact. Example: 'http://localhost:8080' or 'https://cloud.sdu.dk'
# $accessToken is a valid access-token issued by UCloud
# ------------------------------------------------------------------------------------------------------

# In this example, we will show how to create a metadata document and attach it to a file.

# We already have a metadata template in the catalog:

# Authenticated as user
curl -XGET -H "Authorization: Bearer $accessToken" "$host/api/files/metadataTemplates/retrieveLatest?id=15123" 

# {
#     "namespaceId": "sensitivity",
#     "title": "Sensitivity",
#     "version": "1.0.0",
#     "schema": {
#         "type": "object",
#         "title": "UCloud File Sensitivity",
#         "required": [
#             "sensitivity"
#         ],
#         "properties": {
#             "sensitivity": {
#                 "enum": [
#                     "SENSITIVE",
#                     "CONFIDENTIAL",
#                     "PRIVATE"
#                 ],
#                 "type": "string",
#                 "title": "File Sensitivity",
#                 "enumNames": [
#                     "Sensitive",
#                     "Confidential",
#                     "Private"
#                 ]
#             }
#         },
#         "dependencies": {
#         }
#     },
#     "inheritable": true,
#     "requireApproval": true,
#     "description": "File sensitivity for files",
#     "changeLog": "Initial version",
#     "namespaceType": "COLLABORATORS",
#     "uiSchema": {
#         "ui:order": [
#             "sensitivity"
#         ]
#     },
#     "namespaceName": null,
#     "createdAt": 0
# }

# Using this, we can create a metadata document and attach it to our file

curl -XPOST -H "Authorization: Bearer $accessToken" -H "Content-Type: content-type: application/json; charset=utf-8" "$host/api/files/metadata" -d '{
    "items": [
        {
            "fileId": "/51231/my/file",
            "metadata": {
                "templateId": "15123",
                "version": "1.0.0",
                "document": {
                    "sensitivity": "SENSITIVE"
                },
                "changeLog": "New sensitivity"
            }
        }
    ]
}'


# {
#     "responses": [
#         {
#             "id": "651233"
#         }
#     ]
# }

# This specific template requires approval from a workspace admin. We can do this by calling approve.

curl -XPOST -H "Authorization: Bearer $accessToken" -H "Content-Type: content-type: application/json; charset=utf-8" "$host/api/files/metadata/approve" -d '{
    "items": [
        {
            "id": "651233"
        }
    ]
}'


# {
# }

# We can view the metadata by adding includeMetadata = true when requesting any file

curl -XGET -H "Authorization: Bearer $accessToken" "$host/api/files/retrieve?includeOthers=false&includeUpdates=false&includeSupport=false&includeProduct=false&includeMetadata=true&filterHiddenFiles=false&id=51231" 

# {
#     "id": "/51231/my/file",
#     "specification": {
#         "collection": "51231",
#         "product": {
#             "id": "example-ssd",
#             "category": "example-ssd",
#             "provider": "example"
#         }
#     },
#     "createdAt": 1635151675465,
#     "status": {
#         "type": "FILE",
#         "icon": null,
#         "sizeInBytes": null,
#         "sizeIncludingChildrenInBytes": null,
#         "modifiedAt": null,
#         "accessedAt": null,
#         "unixMode": null,
#         "unixOwner": null,
#         "unixGroup": null,
#         "metadata": {
#             "templates": {
#                 "sensitivity": {
#                     "namespaceId": "sensitivity",
#                     "title": "Sensitivity",
#                     "version": "1.0.0",
#                     "schema": {
#                         "type": "object",
#                         "title": "UCloud File Sensitivity",
#                         "required": [
#                             "sensitivity"
#                         ],
#                         "properties": {
#                             "sensitivity": {
#                                 "enum": [
#                                     "SENSITIVE",
#                                     "CONFIDENTIAL",
#                                     "PRIVATE"
#                                 ],
#                                 "type": "string",
#                                 "title": "File Sensitivity",
#                                 "enumNames": [
#                                     "Sensitive",
#                                     "Confidential",
#                                     "Private"
#                                 ]
#                             }
#                         },
#                         "dependencies": {
#                         }
#                     },
#                     "inheritable": true,
#                     "requireApproval": true,
#                     "description": "File sensitivity for files",
#                     "changeLog": "Initial version",
#                     "namespaceType": "COLLABORATORS",
#                     "uiSchema": {
#                         "ui:order": [
#                             "sensitivity"
#                         ]
#                     },
#                     "namespaceName": null,
#                     "createdAt": 0
#                 }
#             },
#             "metadata": {
#                 "sensitivity": [
#                     {
#                         "type": "metadata",
#                         "id": "651233",
#                         "specification": {
#                             "templateId": "15123",
#                             "version": "1.0.0",
#                             "document": {
#                                 "sensitivity": "SENSITIVE"
#                             },
#                             "changeLog": "New sensitivity"
#                         },
#                         "createdAt": 1635151675465,
#                         "status": {
#                             "approval": {
#                                 "type": "approved",
#                                 "approvedBy": "user"
#                             }
#                         },
#                         "createdBy": "user"
#                     }
#                 ]
#             }
#         },
#         "resolvedSupport": null,
#         "resolvedProduct": null
#     },
#     "owner": {
#         "createdBy": "user",
#         "project": null
#     },
#     "permissions": {
#         "myself": [
#             "ADMIN"
#         ],
#         "others": [
#         ]
#     },
#     "updates": [
#     ]
# }

Communication Flow: Visual

Remote Procedure Calls

`browse`

Browses metadata documents

Request Response Error

Request	Response	Error
`FileMetadataBrowseRequest`	`PageV2<FileMetadataAttached>`	`CommonErrorMessage`

FileMetadataBrowseRequest

PageV2<FileMetadataAttached>

CommonErrorMessage

Browses in all accessible metadata documents of a user. These are potentially filtered via the flags provided in the request, such as filtering for specific templates. This endpoint should consider any FileCollection that the user has access to in the currently active project. Note that this endpoint can only return information about the metadata documents and not the file contents itself. Clients should generally present the output of this has purely metadata documents, they can link to the real files if needed. This should eventually result in either a browse or retrieve call in the files API.

`retrieveAll`

Request Response Error

Request	Response	Error
`FileMetadataRetrieveAllRequest`	`FileMetadataRetrieveAllResponse`	`CommonErrorMessage`

FileMetadataRetrieveAllRequest

FileMetadataRetrieveAllResponse

CommonErrorMessage

`approve`

Request Response Error

Request	Response	Error
`BulkRequest<FindByStringId>`	`Unit`	`CommonErrorMessage`

BulkRequest<FindByStringId>

Unit

CommonErrorMessage

`create`

Request Response Error

Request	Response	Error
`BulkRequest<FileMetadataAddRequestItem>`	`BulkResponse<FindByStringId>`	`CommonErrorMessage`

BulkRequest<FileMetadataAddRequestItem>

BulkResponse<FindByStringId>

CommonErrorMessage

`delete`

Request Response Error

Request	Response	Error
`BulkRequest<FileMetadataDeleteRequestItem>`	`Unit`	`CommonErrorMessage`

BulkRequest<FileMetadataDeleteRequestItem>

Unit

CommonErrorMessage

`moveMetadata`

Request Response Error

Request	Response	Error
`BulkRequest<FileMetadataMoveRequestItem>`	`Unit`	`CommonErrorMessage`

BulkRequest<FileMetadataMoveRequestItem>

Unit

CommonErrorMessage

`reject`

Request Response Error

Request	Response	Error
`BulkRequest<FindByStringId>`	`Unit`	`CommonErrorMessage`

BulkRequest<FindByStringId>

Unit

CommonErrorMessage

Data Models

`FileMetadataDocument`

A metadata document which conforms to a FileMetadataTemplate

data class FileMetadataDocument(
    val id: String,
    val specification: FileMetadataDocument.Spec,
    val createdAt: Long,
    val status: FileMetadataDocument.Status,
    val createdBy: String,
    val type: String /* "metadata" */,
)

Properties

`FileMetadataDocument.Spec`

Specification of a FileMetadataDocument

data class Spec(
    val templateId: String,
    val version: String,
    val document: JsonObject,
    val changeLog: String,
)

Properties

`FileMetadataDocument.Status`

The current status of a metadata document

data class Status(
    val approval: FileMetadataDocument.ApprovalStatus,
)

Properties

`FileMetadataDocument.ApprovalStatus`

The approval status of a metadata document

sealed class ApprovalStatus {
    class Approved : ApprovalStatus()
    class NotRequired : ApprovalStatus()
    class Pending : ApprovalStatus()
    class Rejected : ApprovalStatus()
}

`FileMetadataDocument.ApprovalStatus.Approved`

The metadata change has been approved by an admin in the workspace

data class Approved(
    val approvedBy: String,
    val type: String /* "approved" */,
)

Properties

`FileMetadataDocument.ApprovalStatus.Pending`

The metadata document has not yet been approved

data class Pending(
    val type: String /* "pending" */,
)

Properties

`FileMetadataDocument.ApprovalStatus.Rejected`

The metadata document has been rejected by an admin of the workspace

data class Rejected(
    val rejectedBy: String,
    val type: String /* "rejected" */,
)

Properties

`FileMetadataDocument.ApprovalStatus.NotRequired`

The metadata document does not require approval

data class NotRequired(
    val type: String /* "not_required" */,
)

Properties

`FileMetadataAttached`

data class FileMetadataAttached(
    val path: String,
    val metadata: FileMetadataDocument,
)

Properties

`FileMetadataTemplate`

A FileMetadataTemplate allows users to attach user-defined metadata to any UFile

data class FileMetadataTemplate(
    val namespaceId: String,
    val title: String,
    val version: String,
    val schema: JsonObject,
    val inheritable: Boolean,
    val requireApproval: Boolean,
    val description: String,
    val changeLog: String,
    val namespaceType: FileMetadataTemplateNamespaceType,
    val uiSchema: JsonObject?,
    val namespaceName: String?,
    val createdAt: Long?,
)

Properties

`FileMetadataTemplateNamespaceType`

Determines how the metadata template is namespaces

enum class FileMetadataTemplateNamespaceType {
    COLLABORATORS,
    PER_USER,
}

Properties

`FileMetadataAddRequestItem`

data class FileMetadataAddRequestItem(
    val fileId: String,
    val metadata: FileMetadataDocument.Spec,
)

Properties

`FileMetadataBrowseRequest`

The base type for requesting paginated content.

data class FileMetadataBrowseRequest(
    val filterTemplate: String?,
    val filterVersion: String?,
    val filterActive: Boolean?,
    val itemsPerPage: Int?,
    val next: String?,
    val consistency: PaginationRequestV2Consistency?,
    val itemsToSkip: Long?,
)

Paginated content can be requested with one of the following consistency guarantees, this greatly changes the semantics of the call:

Consistency Description

Consistency	Description
`PREFER`	Consistency is preferred but not required. An inconsistent snapshot might be returned.
`REQUIRE`	Consistency is required. A request will fail if consistency is no longer guaranteed.

PREFER

Consistency is preferred but not required. An inconsistent snapshot might be returned.

REQUIRE

Consistency is required. A request will fail if consistency is no longer guaranteed.

The consistency refers to if collecting all the results via the pagination API are consistent. We consider the results to be consistent if it contains a complete view at some point in time. In practice this means that the results must contain all the items, in the correct order and without duplicates.

If you use the PREFER consistency then you may receive in-complete results that might appear out-of-order and can contain duplicate items. UCloud will still attempt to serve a snapshot which appears mostly consistent. This is helpful for user-interfaces which do not strictly depend on consistency but would still prefer something which is mostly consistent.

The results might become inconsistent if the client either takes too long, or a service instance goes down while fetching the results. UCloud attempts to keep each next token alive for at least one minute before invalidating it. This does not mean that a client must collect all results within a minute but rather that they must fetch the next page within a minute of the last page. If this is not feasible and consistency is not required then PREFER should be used.

📝 NOTE: Services are allowed to ignore extra criteria of the request if the next token is supplied. This is needed in order to provide a consistent view of the results. Clients should provide the same criterion as they paginate through the results.

Properties

`FileMetadataDeleteRequestItem`

data class FileMetadataDeleteRequestItem(
    val id: String,
    val changeLog: String,
)

Properties

`FileMetadataMoveRequestItem`

data class FileMetadataMoveRequestItem(
    val oldFileId: String,
    val newFileId: String,
)

Properties

`FileMetadataRetrieveAllRequest`

data class FileMetadataRetrieveAllRequest(
    val fileId: String,
)

Properties

`FileMetadataRetrieveAllResponse`

data class FileMetadataRetrieveAllResponse(
    val metadata: List<FileMetadataAttached>,
)

Properties

Previousmetadata Nexttemplates

Last updated 6 days ago

SDUCloud

documents

Metadata Documents

Rationale

Table of Contents

Example: Sensitivity Document

Remote Procedure Calls

`browse`

`retrieveAll`

`approve`

`create`

`delete`

`moveMetadata`

`reject`

Data Models

`FileMetadataDocument`

`FileMetadataDocument.Spec`

`FileMetadataDocument.Status`

`FileMetadataDocument.ApprovalStatus`

`FileMetadataDocument.ApprovalStatus.Approved`

`FileMetadataDocument.ApprovalStatus.Pending`

`FileMetadataDocument.ApprovalStatus.Rejected`

`FileMetadataDocument.ApprovalStatus.NotRequired`

`FileMetadataAttached`

`FileMetadataTemplate`

`FileMetadataTemplateNamespaceType`

`FileMetadataAddRequestItem`

`FileMetadataBrowseRequest`

`FileMetadataDeleteRequestItem`

`FileMetadataMoveRequestItem`

`FileMetadataRetrieveAllRequest`

`FileMetadataRetrieveAllResponse`