{"type":"doc","content":[{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"toc","parameters":{"macroParams":{"minLevel":{"value":"1"},"maxLevel":{"value":"2"},"outline":{"value":"false"},"type":{"value":"list"},"printable":{"value":"false"}},"macroMetadata":{"macroId":{"value":"91607632-feca-4ebf-bb48-d1e4d10784eb"},"schemaVersion":{"value":"1"},"title":"Table of Contents"}},"localId":"406eb015-7492-47ef-b9cf-2ee095a5b892"}},{"type":"paragraph","content":[{"text":"Custom metadata headers are created as a means to pass data required by applications. Including custom metadata on stored objects increases the usefulness of content: it provides information that can be indexed by Elasticsearch and used to find, filter, and analyze the content later.","type":"text"}]},{"type":"panel","attrs":{"panelType":"note"},"content":[{"type":"heading","attrs":{"level":3},"content":[{"text":"Note","type":"text"}]},{"type":"paragraph","content":[{"text":"Swarm stores these headers and supplied values without parsing, validation, or modification.","type":"text"}]}]},{"type":"paragraph","content":[{"text":"Work with custom metadata through the WRITE, UPDATE, and COPY methods. The COPY method allows updating and adding to the metadata on objects ","type":"text"},{"text":"after","type":"text","marks":[{"type":"em"}]},{"text":" the initial WRITE.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"heading","attrs":{"level":3},"content":[{"text":"Tip","type":"text"}]},{"type":"paragraph","content":[{"text":"With COPY requests, add the ","type":"text"},{"text":"preserve","type":"text","marks":[{"type":"strong"}]},{"text":" ","type":"text"},{"text":"query argument","type":"text","marks":[{"type":"link","attrs":{"href":"/wiki/spaces/public/pages/2443821292"}}]},{"text":" to guarantee any custom metadata existing on the object is carried over to the copy. To overwrite an existing value, include the header name with the new value on the request. (v9.2)","type":"text"}]}]},{"type":"paragraph","content":[{"text":"See ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://perifery.atlassian.net/wiki/spaces/public/pages/2443820996"}},{"text":".","type":"text"}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Requirements for Custom Names","type":"text"}]},{"type":"paragraph","content":[{"text":"Characters:","type":"text","marks":[{"type":"strong"}]},{"text":" For best compatibility going forward, Swarm restricts to these characters in custom metadata header names (v9.1):","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"letters (both cases, although case-insensitive is consistent with ","type":"text"},{"text":"HTTP/1.1 RFC","type":"text","marks":[{"type":"textColor","attrs":{"color":"#0070c0"}},{"type":"link","attrs":{"href":"http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2"}}]},{"text":")","type":"text","marks":[{"type":"textColor","attrs":{"color":"#0070c0"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"numbers","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"dash (hyphen)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"underscore","type":"text"}]}]}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"heading","attrs":{"level":3},"content":[{"text":"Elasticsearch and Dots","type":"text"}]},{"type":"paragraph","content":[{"text":"Some versions of Elasticsearch (such as 2.3.3) do not allow dots in normal field names. When indexing objects, Swarm converts any dots in custom metadata field names (","type":"text"},{"text":"x_foo_meta_2016.12","type":"text","marks":[{"type":"code"}]},{"text":") to underscores (","type":"text"},{"text":"x_foo_meta_2016_12","type":"text","marks":[{"type":"code"}]},{"text":"). (v9.1)","type":"text"}]}]},{"type":"paragraph","content":[{"text":"Formats: ","type":"text","marks":[{"type":"strong"}]},{"text":"Follow one of these two naming formats when custom headers are defined, or they are silently ignored and not persisted to the storage cluster: ","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"x-*-","type":"text","marks":[{"type":"textColor","attrs":{"color":"#595959"}},{"type":"strong"}]},{"text":"meta","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"x-*-","type":"text","marks":[{"type":"textColor","attrs":{"color":"#595959"}},{"type":"strong"}]},{"text":"meta","type":"text","marks":[{"type":"strong"}]},{"text":"-*","type":"text"}]}]}]},{"type":"codeBlock","content":[{"text":"x-ExampleCorp-meta-color: blue","type":"text"}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Requirements for Custom Values","type":"text"}]},{"type":"paragraph","content":[{"text":"To specify more than one value for the same header, list the values on the same line, separated by commas.","type":"text"}]},{"type":"codeBlock","content":[{"text":"x-color-meta: blue, green","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"heading","attrs":{"level":3},"content":[{"text":"Important","type":"text"}]},{"type":"paragraph","content":[{"text":"Do ","type":"text"},{"text":"not","type":"text","marks":[{"type":"em"}]},{"text":" reuse the same header with different values.","type":"text"}]}]},{"type":"paragraph","content":[{"text":"For metadata values, use 7-bit US-ASCII characters, or else follow ","type":"text"},{"text":"RFC 2047","type":"text","marks":[{"type":"link","attrs":{"href":"https://www.ietf.org/rfc/rfc2047.txt"}}]},{"text":" guidelines for alternate character sets.","type":"text"}]},{"type":"codeBlock","content":[{"text":"x-xml-meta-data: largeblue","type":"text"}]},{"type":"paragraph","content":[{"text":"Verify the total length of ","type":"text"},{"text":"all","type":"text","marks":[{"type":"em"}]},{"text":" persisted metadata, keys and values, does not exceed 32 KB. Metadata over 32 KB results in a ","type":"text"},{"text":"400 Bad Request ","type":"text","marks":[{"type":"strong"}]},{"text":"error response from Swarm.","type":"text"}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Sample Scenario for Custom Metadata","type":"text"}]},{"type":"paragraph","content":[{"text":"Assume a domain of \"","type":"text"},{"text":"example.com","type":"text","marks":[{"type":"code"}]},{"text":"\" with a bucket called \"","type":"text"},{"text":"surveillance","type":"text","marks":[{"type":"code"}]},{"text":"\", created for storing the company's surveillance videos. ","type":"text"}]},{"type":"paragraph","content":[{"text":"To add a video, POST to the bucket, specifying the Content-Type of the video and including custom metadata to document the video's duration, camera location, and camera model:","type":"text"}]},{"type":"codeBlock","attrs":{"language":"shell"},"content":[{"text":"curl -i --location-trusted -X POST --post301\t\\\n\t--data-binary @20170311-972-9928817883.mp4 \t\\\n\t-H \"Expect: 100-continue\" \t\t\t\t\t\\\n\t-H \"x-example-meta-Start-Time: 2017-03-11T12:00:01.678Z\" \\\n\t-H \"x-example-meta-End-Time: 2017-03-11T13:00:00.421Z\" \t\\\n\t-H \"x-example-meta-Building: Annex 2\" \\\n\t-H \"x-example-meta-Location: 972\" \\\n\t-H \"x-example-meta-CameraModel: SWDSK-850004A-US\" \\\n\t-H \"Content-Type: video/mp4\" \\\n\t-H \"Content-Disposition: inline\" \\\n\t\"http://example.com/surveillance/2017/03/22/20170311-972-9928817883.mp4\"\n\nHTTP/1.1 100 Continue\nDate: Mon, 27 Mar 2017 17:15:26 GMT\nServer: CAStor Cluster/9.2.0\nContent-Length: 0\n\nHTTP/1.1 201 Created\nLocation: http://192.168.1.12:80/surveillance/2017/03/22/20170311-972-9928817883.mp4 \\\n\t?domain=example.com\nLocation: http://192.168.1.13:80/surveillance/2017/03/22/20170311-972-9928817883.mp4 \\\n\t?domain=example.com\nVolume: 8aff01dbe86d6ff1f27b5872bfc8e840\nVolume: cef223aa1bfc13e356203fdede8489e4\nManifest: ec\nLast-Modified: Mon, 27 Mar 2017 17:15:25 GMT\nCastor-System-Encoding: zfec 1.4(2, 1, 524288, 200000000)\nCastor-System-Version: 1490634925.750\nEtag: \"c04b7eac90a3f22292581080c32fdd07\"\nReplica-Count: 2\nDate: Mon, 27 Mar 2017 17:17:16 GMT\nServer: CAStor Cluster/9.2.0\nContent-Length: 46\nContent-Type: text/html\nKeep-Alive: timeout=14400\nNew stream created","type":"text"}]},{"type":"paragraph","content":[{"text":"Use a HEAD command to verify the video is successfully stored:","type":"text"}]},{"type":"codeBlock","attrs":{"language":"shell"},"content":[{"text":"curl --head \\\n\t--location-trusted \"http://example.com/surveillance/2017/03/22/20170311-972-9928817883.mp4\"\n\nHTTP/1.1 301 Moved Permanently\nDate: Mon, 27 Mar 2017 17:22:50 GMT\nServer: CAStor Cluster/9.2.0\nLocation: http://192.168.1.12:80/surveillance/2017/03/22/20170311-972-9928817883.mp4\n\t?domain=example.com&auth=2db96e4590e029966aecfd0dd96da7e9\nContent-Length: 0\nKeep-Alive: timeout=14400\n\nHTTP/1.1 200 OK\nCastor-System-CID: fd20ce977b35d0509205b27977d697d3\nCastor-System-Cluster: example.com\nCastor-System-Created: Mon, 27 Mar 2017 17:15:25 GMT\nCastor-System-Name: 2017/03/22/20170311-972-9928817883.mp4\nCastor-System-Version: 1490634925.750\nContent-Disposition: inline\nContent-Type: video/mp4\nLast-Modified: Mon, 27 Mar 2017 17:15:25 GMT\nx-example-meta-Building: Annex 2\nx-example-meta-CameraModel: SWDSK-850004A-US\nx-example-meta-End-Time: 2017-03-11T13:00:00.421Z\nx-example-meta-Location: 972\nx-example-meta-Start-Time: 2017-03-11T12:00:01.678Z\nManifest: ec\nContent-Length: 1500964975\nEtag: \"c04b7eac90a3f22292581080c32fdd07\"\nCastor-System-Path: /example.com/surveillance/2017/03/22/20170311-972-9928817883.mp4\nCastor-System-Domain: example.com\nVolume: 8aff01dbe86d6ff1f27b5872bfc8e840\nDate: Mon, 27 Mar 2017 17:22:50 GMT\nServer: CAStor Cluster/9.2.0\nKeep-Alive: timeout=14400","type":"text"}]},{"type":"paragraph","content":[{"text":"The custom metadata is what makes it possible and practical to find videos. Suppose that an incident occurred in the Annex 2 building; to find surveillance video that may be relevant to the investigation, search the ","type":"text"},{"text":"surveillance","type":"text","marks":[{"type":"strong"}]},{"text":" bucket for video taken at Annex 2 during that time span:","type":"text"}]},{"type":"codeBlock","attrs":{"language":"shell"},"content":[{"text":"curl -i --location-trusted \"http://192.168.1.11/surveillance\\\n\t?domain=example.com\\\n\t&format=json&fields=all\\\n\t&content-type=video/mp4\\\n\t&x-example-meta-Building=Annex%202\\\n\t&x-example-meta-Start-Time:date=<2017-03-11T12:17:23Z\\\n\t&x-example-meta-End-Time:date=>2017-03-11T12:17:23Z\"\n\nHTTP/1.1 200 OK\nCastor-System-Alias: fd20ce977b35d0509205b27977d697d3\nCastor-System-CID: 72203a85b0f9d7a64a7625c114f8a886\nCastor-System-Cluster: example.com\nCastor-System-Created: Mon, 27 Mar 2017 16:37:38 GMT\nCastor-System-Name: surveillance\nCastor-System-Version: 1490632658.361\nX-Timestamp: Mon, 27 Mar 2017 16:37:38 GMT\nLast-Modified: Mon, 27 Mar 2017 17:26:00 GMT\nTransfer-Encoding: chunked\nContent-Type: application/json; charset=utf-8\nCastor-Object-Count: 1\nCastor-System-Object-Count: 1\nDate: Mon, 27 Mar 2017 17:26:00 GMT\nServer: CAStor Cluster/9.2.0\nKeep-Alive: timeout=14400\n[{\n\t\"sizewithreps\": 2251447463,\n\t\"contextid\": \"fd20ce977b35d0509205b27977d697d3\",\n\t\"content_type\": \"video/mp4\",\n\t\"name\": \"2017/03/22/20170311-972-9928817883.mp4\",\n\t\"x_example_meta_end_time:date\": \"2017-03-11T13:00:00.421Z\",\n\t\"@timestamp\": 1490635036512,\n\t\"x_example_meta_building\": \"Annex 2\",\n\t\"x_example_meta_location:date\": 972000,\n\t\"x_example_meta_location\": \"972\",\n\t\"x_example_meta_cameramodel\": \"SWDSK-850004A-US\",\n\t\"domainid\": \"72203a85b0f9d7a64a7625c114f8a886\",\n\t\"x_example_ meta_start_time:date\": \"2017-03-11T12:00:01.678Z\",\n\t\"hash\": \"c04b7eac90a3f22292581080c32fdd07\",\n\t\"timestamp\": 1490635036512,\n\t\"x_example_meta_location:double\": 972,\n\t\"last_modified\": \"2017-03-27T17:15:25.748400Z\",\n\t\"bytes\": 1500964975,\n\t\"content_disposition\": \"inline\",\n\t\"x_example_meta_location:long\": 972,\n\t\"x_example_me ta_end_time\": \"2017-03-11T13:00:00.421Z\",\n\t\"x_example_meta_start_time\": \"2017-03-11T12:00:01.678Z\"\n}]","type":"text"}]},{"type":"paragraph","content":[{"text":"The search correctly found a video of interest in the ","type":"text"},{"text":"surveillance","type":"text","marks":[{"type":"strong"}]},{"text":" bucket and returned the object: ","type":"text"},{"text":"2017/03/22/20170311-972-9928817883.mp4","type":"text","marks":[{"type":"code"}]}]}],"version":1}

Browser not supported