commit 324a82a959f2adb64a000d610d1180646e8b23f5 Author: Allison Karlitskaya Date: Thu May 29 01:54:11 2025 +0200 Initial commit Signed-off-by: Allison Karlitskaya diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..042776a --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/Cargo.lock +/target/ diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..8cf8fe2 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "zstd-chunked" +version = "0.1.0" +authors = ["Allison Karlitskaya "] +description = "Read zstd:chunked files" +edition = "2024" +keywords = ["containers", "zstd", "zstd-chunked"] +license = "MIT OR Apache-2.0" +readme = "README.md" +repository = "https://github.com/containers/zstd-chunked-rs" + +[dependencies] +anyhow = "1.0.98" +zerocopy = { version = "0.8.25", features = ["derive"] } +zstd = "0.13.3" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +base64 = "0.22.1" + +[dev-dependencies] +clap = { version = "4.5.39", features = ["derive"] } + +[lints.rust] +missing_docs = "forbid" +unsafe_code = "forbid" +warnings = "deny" + +[lints.clippy] +all = "deny" +expect_used = "deny" +nursery = "deny" +panic = "deny" +pedantic = "deny" +unwrap_used = "deny" diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 0000000..8f71f43 --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 0000000..31aa793 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,23 @@ +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..c8dd6ca --- /dev/null +++ b/README.md @@ -0,0 +1,22 @@ +# zstd-chunked-rs + +This is a simple library to help read `zstd:chunked`-encoded files. + +There's also [file format documentation here](./docs/format.md). + +## License + +Licensed under either of + + * Apache License, Version 2.0 + ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) + * MIT license + ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) + +at your option. + +## Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the Apache-2.0 license, shall be +dual licensed as above, without any additional terms or conditions. diff --git a/docs/format.md b/docs/format.md new file mode 100644 index 0000000..38c3240 --- /dev/null +++ b/docs/format.md @@ -0,0 +1,241 @@ +# zstd:chunked file format + +This is a rough documentation of the zstd:chunked file format. It's a mix of several formats: + + - [tar](https://en.wikipedia.org/wiki/.tar) + - [zstd](https://github.com/facebook/zstd) + - [tar-split](https://github.com/vbatts/tar-split/) + - [CRFS Table of Contents JSON](https://github.com/google/crfs/blob/71d77da419c90be7b05d12e59945ac7a8c94a543/stargz/stargz.go#L108) + - a "footer" + - a set of annotations on [OCI Descriptors](https://github.com/opencontainers/image-spec/blob/main/descriptor.md) + +The main concept is of a zstd compressed `.tar.zstd` file which, when normally decompressed, produces the desired layer +`.tar`, with the correct checksum. This mode of operations is only intended to be used by clients which are unaware of +`zstd:chunked`, however. + +For clients that are aware of the format, `zstd:chunked` takes advantage of the fact that (like tar and gzip) a zstd +file can consist of multiple independent concatenated frames. It also takes advantage of [zstd Skippable +Frames](https://datatracker.ietf.org/doc/html/rfc8878#name-skippable-frames) to store some metadata at the end of the +file which can be used for finding the frames that we need to download. Finally, [HTTP Range +Requests](https://www.rfc-editor.org/rfc/rfc7233) are used to only download the frames that we actually require. + +This document is written from the standpoint of someone trying to consume the format. + +# Overall layout + +The `.tar.zstd` file will be compressed as a large number of separate frames, such that each individual non-empty +regular file has its content contained within a single frame. This typically implies that the tar headers between +regular files will also be separated into their own frames. + +At the end of the file are three skippable frames: + - the compressed "manifest" JSON + - the compressed "tarsplit" JSON + - the uncompressed "footer" + +In order to do anything useful with the file you first need to locate the manifest and tarsplit JSON files in the +compressed stream. + +# Finding the skippable frames + +Having the skippable frames at the end of file makes things a bit more difficult: it's generally not possible to find +frames by scanning backwards. There are two ways to solve this problem: + - the fixed-sized uncompressed footer + - OCI descriptor annotations + +## The footer + +The footer is a fixed-sized uncompressed skip-frame, 64 bytes in content length (72 bytes total length). A skippable +frame header starts with a 32bit little endian magic number (as per the zstd spec, "any value from 0x184D2A50 to +0x184D2A5F") but we always use `0x184d2a50`. It's followed by a 32bit little endian size (which is always 64 in our +case). + +The content of the footer content consists of the following 64bit little endian integers, in order: + - manifest start offset, compressed length, uncompressed length + - manifest type (always 1) + - tarsplit start offset, compressed length, uncompressed length + - magic number: 0x78556e496c554e47 (ie: the ascii string "GNUlInUx") + +That means that any zstd:chunked file will always end with 72 bytes which look something like: + +``` +00000000 50 2a 4d 18 40 00 00 00 6c 91 62 06 00 00 00 00 |P*M.@...l.b.....| +00000010 9e 68 0f 00 00 00 00 00 e7 4e 54 00 00 00 00 00 |.h.......NT.....| +00000020 01 00 00 00 00 00 00 00 12 fa 71 06 00 00 00 00 |..........q.....| +00000030 e2 57 09 00 00 00 00 00 76 07 eb 00 00 00 00 00 |.W......v.......| +00000040 47 4e 55 6c 49 6e 55 78 |GNUlInUx| +``` + + - `50 2a 4d 18 40 00 00 00`: skippable frame, size 0x40 (64) + - `6c 91 62 06 00 00 00 00`: start of the manifest in the compressed stream + - `9e 68 0f 00 00 00 00 00`: length of the manifest in the compressed stream + - `e7 4e 54 00 00 00 00 00`: uncompressed size of the manifest + - `01 00 00 00 00 00 00 00`: manifest type (1) + - `12 fa 71 06 00 00 00 00`: start of the tarsplit json in the compressed stream + - `e2 57 09 00 00 00 00 00`: length of the tarsplit json in the compressed stream + - `76 07 eb 00 00 00 00 00`: uncompressed size of the tarsplit json + - `47 4e 55 6c 49 6e 55 78`: magic number (`GNUlInUx`) + +## The OCI descriptor annotations + +Of course, you're probably interested in downloading the layer because it's part of an OCI image. In that case, the +same information that's contained in the header will have also been encoded as a set of annotations on the descriptor +for the layer: + +```json + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+zstd", + "digest": "sha256:20574ef181bd366aa8a344c3e869c95a22429feb00f1c4feeb7fb2fd0e8de71c", + "size": 108745276, + "annotations": { + "io.github.containers.zstd-chunked.manifest-checksum": "sha256:44b5219a19eea4bd8414c2938d0561eebdd053fad8110df7957bee86370ba02b", + "io.github.containers.zstd-chunked.manifest-position": "107123052:1009822:5525223:1", + "io.github.containers.zstd-chunked.tarsplit-checksum": "sha256:4041c7b1197a991b90eb5e933c3e96e5f60abc56ed4c0bc926a0d5a2e136ebdc", + "io.github.containers.zstd-chunked.tarsplit-position": "108132882:612322:15402870" + } + } +``` + +The annotations are: + + - `io.github.containers.zstd-chunked.manifest-checksum`: a digest of the compressed "manifest" JSON + - `io.github.containers.zstd-chunked.manifest-position`: a `:`-separated 4-tuple containing the manifest location + information in the same order as it appears in the footer: offset, compressed size, uncompressed size, manifest type + - `io.github.containers.zstd-chunked.tarsplit-checksum`: a digest of the compressed "tarsplit" JSON + - `io.github.containers.zstd-chunked.tarsplit-position`: a `:`-separated 3-tuple containing the tarsplit location + information in the same order as it appears in the footer: offset, compressed size, uncompressed size + +# The "manifest" file format + +The manifest is obtained by slicing (or fetching) the manifest start offset up to the specified compressed length from +the original file. That range contains a single normal compressed zstd frame which you decompress to get the manifest. +As mentioned above, the manifest is contained inside of a skippable frame, but the offsets for finding the manifest do +not include the skippable frame header, so you don't need to do anything about it. + +This file format was originally designed as part of the [Container Registry Filesystem](https://github.com/google/crfs/) +project and contains far more information than is required for incremental downloading. It more or less duplicates all +of the tar header information in a different format. It was designed to allow a lazy-downloading filesystem +implementation that could do metadata lookups without having to fetch tar headers that were scattered around the rest of +the file. We can safely ignore most of it. + +At the top-level, it's a JSON dictionary containing two items, and looking something like: + +```json +{ + "version": 1, + "entries": [ + { + "type": "dir", + "name": "etc/", + "mode": 493, + "modtime": "2025-05-28T02:24:19Z", + "accesstime": "0001-01-01T00:00:00Z", + "changetime": "0001-01-01T00:00:00Z", + "xattrs": { + "user.overlay.impure": "eQ==" + } + }, + { + "type": "reg", + "name": "etc/asound.conf", + "mode": 420, + "size": 55, + "modtime": "2025-04-14T00:00:00Z", + "accesstime": "0001-01-01T00:00:00Z", + "changetime": "0001-01-01T00:00:00Z", + "digest": "sha256:3b7fe1f8fd9bb7e1c80fddd31c592bc5b7b7d39e322ded8303c742b2bc1bec31", + "offset": 210, + "endOffset": 278 + } + ] +} +``` + +The `version` is 1. + +`entries` is an array. Each entry is either a complete description of an entry in the `.tar` file (in which case the +type will be `"reg"`, `"dir"`, `"hardlink"`, `"symlink"`, etc.) or an additional "chunk" (type of `"chunk"`). Chunk +entries follow the file which is chunked, allowing it to be split into smaller chunks (which may improve incremental +downloads). It's possible to ignore chunks when implementing the file format: even in the presence of chunks, the +information required to download the complete data of the file is available on the main entry: you'll simply get it as +multiple concatenated zstd frames (which, as mentioned above, is still a valid compressed file). + +The important fields for knowing what needs to be downloaded are `"digest"`, `"offset"`, and `"endOffset"`. The digest +provides the main mechanism for knowing if we already have a particular file's contents downloaded and the offsets +provide us with the information we need to fetch it if we don't have it. The `"digest"` field is a digest of the +uncompressed file data. The `"offset"` is the offset of the start of the compressed frame and the `"endOffset"` is the +offset past the end of the frame (ie: you need to subtract 1 from this when turning it into a Range request). `"size"` +is the size of the uncompressed data, and it's useful for verification. + +After decompressing, the data from the given range does not contain any extra padding which might be implied by the tar +format (ie: rounding up to the next 512-byte block). The uncompressed data should match the `"size"` and `"digest"`. + +For purposes of incremental downloads, we really only need the entries of `"type": "reg"` with a non-zero `"size"` and +`"digest"` plus `"offset`" and `"endOffset"`. Those are the entries that will let us find our file content. + +## Chunks + +The `"type": "chunk"` entries contain information about individual file chunks. It's not specified which algorithm is +used to determine the chunking, but [containers/storage](https://github.com/containers/storage) uses a rolling checksum +approach. In an addition to the original format, zstd:chunked can also contain `"chunkType": "zeros"` chunks which are +effectively sparse "holes". + +The chunk format is not described here because it's not implemented yet: the most obvious approach to doing so would +require duplicating file content on disk (chunked and merged) and it's not clear if it's worth it to save a bit of extra +downloading. + +# The "tarsplit" file format + +This is the JSON-lines format produced by the [tar-split](https://github.com/vbatts/tar-split/) utility used in the +implementation of podman, explaining its presence in this file format. + +The purpose of this format is to store enough metadata to allow reconstructing the original content of a `.tar` file +which was previously unpacked, assuming we still have the unpacked contents around (referred to by their filenames). + +The inclusion of this file essentially reproduces all of the data in the tar headers for a third time. + +It looks something like this: + +```json +{"type":2,"payload":"ZXRjL1BheEhlYWRlcnMuMAAA...","position":0} +{"type":1,"name":"etc/","payload":null,"position":1} +{"type":2,"payload":"ZXRjL2Fzb3VuZC5jb25mAAA...=","position":2} +{"type":1,"name":"etc/asound.conf","size":55,"payload":"tAt3+IpQDrE=","position":3} +{"type":2,"payload":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA...==","position":4} +``` + +where each line contains one dictionary which is either `"type": 1` or `"type": 2`. + +The "type 1" items are references to file content. In the originally-conceived use of the tool, these would be files +that you could read from the filesystem to get the content to include back in the reconstructed tar stream. In our +case, we can find those files in the manifest (along with the information about where we can fetch them and if we need +to). The important key here is `"name"` which is the filename: it's exactly the same name as in the manifest. These +entries are really only useful for regular files. You can identify that case by the presence of a `"size"` field: the +uncompressed file size (which should match the same key in the manifest). The `"payload"` field on "type 1" entries +contains a base64-encoded crc64 of the file contents. + +The "type 2" items are inline content used to encode non-file data. The tar headers end up reproduced in these. It's +worth noting that padding is *included* in these items. That means that the payload of a "type 2" entry following a +"type 1" entry that doesn't have a multiple-of-512 file size will start with padding (seen as "AAAA..." in the above +example). + +# Putting it all together + +The first step is to get the compressed form of the manifest and the tarsplit JSON files. You can use the OCI +annotations if you have them, or do a "suffix" range request for the footer. + +Once you have the two JSON files you need to decompress them. + +Rebuilding the `.tar` file contents then essentially works by iterating over the lines in the tarsplit file. For each +"type 2" entry, you simply output the inline data. For each "type 1" entry, you look up the equivalent entry in the +manifest, check if you already have an object with that checksum, use the range information to download it if not, and +then output its content. + +In all cases I've seen, the entries in the tarstream and the manifest are in the same order, so it's probably possible +to create an efficient implementation that decompresses them in parallel, keeping them in sync with each other, avoiding +ever having to fully extract either file at the same time. In practice, the total amount of data in the manifest is +relatively small, and extracting it to some form of a lookup table is probably a more reliable (and easier) approach. + +Making individual HTTP requests for each range as we require it is probably not a reasonable idea: there's too much +overhead and latency. HTTP/2 and HTTP/3 improve things by allowing massive levels of concurrency over a single +connection (saving handshaking overhead) but the overhead of the headers is still significant. A better approach is to +pre-filter the list of digests, removing the ones we already have, and fetching many ranges per request. diff --git a/examples/extract.rs b/examples/extract.rs new file mode 100644 index 0000000..0514308 --- /dev/null +++ b/examples/extract.rs @@ -0,0 +1,43 @@ +//! Extracts a zstd:chunked file to stdout one chunk at a time +//! Should produce the exact same output as `zstdcat` on the same file + +use core::ops::Range; + +use anyhow::{Context, Result}; +use clap::Parser; + +#[derive(Parser)] +struct Args { + /// A zstd:chunked file to extract + filename: String, +} + +use zstd_chunked::{MetadataReferences, Stream}; + +fn ref_from_slice<'a>(data: &'a [u8], range: &Range) -> Result<&'a [u8]> { + let start = usize::try_from(range.start)?; + let end = usize::try_from(range.end)?; + data.get(start..end).context("Out of range!") +} + +fn print_zstd_chunked(data: &[u8]) -> Result<()> { + let references = MetadataReferences::from_footer(data) + .context("This doesn't appear to be a zstd:chunked file")?; + + let stream = Stream::new_from_frames( + ref_from_slice(data, &references.manifest.range)?, + ref_from_slice(data, &references.tarsplit.range)?, + )?; + + stream.write_to(&mut std::io::stdout(), |reference| { + Ok(zstd::decode_all(ref_from_slice(data, &reference.range)?)?) + })?; + + Ok(()) +} + +fn main() -> Result<()> { + let args = Args::parse(); + let content = std::fs::read(args.filename).context("Unable to open file")?; + print_zstd_chunked(&content).context("Failed to process zstd:chunked file?") +} diff --git a/src/format.rs b/src/format.rs new file mode 100644 index 0000000..e8cdcd3 --- /dev/null +++ b/src/format.rs @@ -0,0 +1,99 @@ +use anyhow::Result; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as b64; +use serde::{ + Deserialize, + de::{self, Deserializer}, +}; +use zerocopy::{ + FromBytes, Immutable, IntoBytes, KnownLayout, Unaligned, + little_endian::{U32, U64}, +}; + +// "tarsplit" file format +#[derive(Debug, Deserialize)] +pub struct TarSplitEntry { + #[serde(default)] + pub name: Option, + #[serde(default)] + pub size: Option, + #[serde(default)] + #[serde(deserialize_with = "deserialize_option_base64")] + pub payload: Option>, +} + +fn deserialize_option_base64<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + let opt = Option::::deserialize(deserializer)?; + opt.map_or_else( + || Ok(None), + |s| { + b64.decode(&s) + .map(Vec::into_boxed_slice) + .map(Some) + .map_err(de::Error::custom) + }, + ) +} + +// "manifest" file format +#[derive(Debug, Deserialize)] +pub struct Manifest { + pub version: u32, + pub entries: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct ManifestEntry { + pub name: String, + pub size: Option, + pub digest: Option, + pub offset: Option, + #[serde(rename = "endOffset")] + pub end_offset: Option, +} + +// Footer +#[repr(C)] +#[derive(Debug, Copy, Clone, IntoBytes, FromBytes, Unaligned, KnownLayout, Immutable)] +pub struct FooterReference { + pub offset: U64, + pub length_compressed: U64, + pub length_uncompressed: U64, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone, IntoBytes, FromBytes, Unaligned, KnownLayout, Immutable)] +pub struct Footer { + pub(crate) skippable_magic: [u8; 4], + pub(crate) skippable_size: U32, + + pub(crate) manifest: FooterReference, + pub(crate) manifest_type: U64, + pub(crate) tarsplit: FooterReference, + + pub(crate) zstd_chunked_magic: [u8; 8], +} + +const ZSTD_SKIPPABLE_MAGIC: [u8; 4] = [0x50, 0x2a, 0x4d, 0x18]; +const ZSTD_CHUNKED_FOOTER_SIZE: u32 = 64; +const ZSTD_CHUNKED_MANIFEST_TYPE: u64 = 1; +const ZSTD_CHUNKED_MAGIC: [u8; 8] = *b"GNUlInUx"; + +impl Footer { + fn valid(&self) -> bool { + self.skippable_magic == ZSTD_SKIPPABLE_MAGIC + && self.skippable_size == ZSTD_CHUNKED_FOOTER_SIZE + && self.manifest_type == ZSTD_CHUNKED_MANIFEST_TYPE + && self.zstd_chunked_magic == ZSTD_CHUNKED_MAGIC + } + + /// Tries to extract a zstd:chunked footer from the passed slice. The slice can be the entire + /// file or some portion of the end of it, but should be at least 72 bytes in length. + pub fn from_suffix(data: &[u8]) -> Option<&Self> { + let (_rest, footer) = Self::ref_from_suffix(data).ok()?; + if footer.valid() { Some(footer) } else { None } + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..bfd9f92 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,237 @@ +//! A library to help read zstd:chunked files +mod format; + +use core::ops::Range; +use std::{ + collections::HashMap, + io::{Read, Write}, +}; + +use anyhow::{Context, Result, ensure}; +use zstd::stream::Decoder; + +use self::format::{Footer, FooterReference, Manifest, TarSplitEntry}; + +/// A reference to a compressed range in a zstd:chunked file, along with size and checksum +/// information about the uncompressed data at that range. +#[derive(Debug, Clone)] +pub struct ContentReference { + /// The range itself, in bytes, in the compressed file. + pub range: Range, + + /// The digest of the data at the range, after decompression. + pub digest: String, + + /// The size of the compressed data at the range, after decompression. + pub size: u64, +} + +/// A chunk of data in a zstd:chunked stream. Either contains inline data or a reference to a +/// compressed range (and checksum and size information about the data at that range). +#[derive(Debug, Clone)] +pub enum Chunk { + /// The literal data appears directly. + Inline(Box<[u8]>), + /// The data appears at the referenced range, which may need to be fetched and decompressed. + External(Box<[ContentReference]>), +} + +/// Represents the layout of a zstd:chunked file. You can reconstruct the original file contents +/// by iterating over the chunks. +#[derive(Debug)] +pub struct Stream { + /// The chunks in the file. + pub chunks: Vec, +} + +impl Stream { + /// Create the metadata structure from the compressed frames referred to by the ranges in the + /// OCI layer descriptor annotations or the file footer. + /// + /// # Errors + /// + /// This function can fail if any of the metadata isn't in the expected format (zstd-compressed + /// JSON) or if there are missing mandatory fields or internal inconsistencies. In all cases, + /// it indicates a corrupt zstd:chunked file (or a bug in the library). + pub fn new_from_frames(manifest: impl Read, tarsplit: impl Read) -> Result { + let manifest: Manifest = serde_json::from_reader(Decoder::new(manifest)?)?; + + ensure!( + manifest.version == 1, + "Incorrect zstd:chunked CRFS manifest version" + ); + + // Read the manifest entries into a table by filename, taking only the ones that have the + // digest, size, offset and end_offset information filled in (ie: regular files). Don't + // handle chunks. + let manifest_entries: HashMap = manifest + .entries + .into_iter() + .filter_map(|entry| { + Some(( + entry.name, + ContentReference { + digest: entry.digest?, + size: entry.size?, + range: entry.offset?..entry.end_offset?, + }, + )) + }) + .collect(); + + // Iterate over the chunks in the tarsplit. For inline chunks, store the inline data. For + // external chunks, look them up in the manifest_entries and store what we find. + let chunks = serde_json::Deserializer::from_reader(Decoder::new(tarsplit)?) + .into_iter() + .map(|entry| { + Ok(match entry? { + TarSplitEntry { + name: Some(name), + size: Some(size), + .. // ignored: crc64 + } => { + let reference = manifest_entries.get(&name) + .with_context(|| format!("Filename {name} in zstd:chunked tarsplit missing from manifest"))?; + ensure!(size == reference.size, "size mismatch"); + Some(Chunk::External(Box::from([reference.clone()]))) + } + TarSplitEntry { + payload: Some(payload), + .. + } => Some(Chunk::Inline(payload)), + _ => None, + }) + }) + .filter_map(Result::transpose) + .collect::>>()?; + + Ok(Self { chunks }) + } + + /// Iterates over all of the references that need to be satisfied for this stream to be + /// reconstructed. This might be useful to help prefetch the required items. + pub fn references(&self) -> impl Iterator { + self.chunks.iter().flat_map(|chunk| { + if let Chunk::External(items) = chunk { + items.as_ref() + } else { + &[] + } + }) + } + + /// Writes the content of the stream to the given writer. The `resolve_reference()` function + /// should return the *decompressed* data corresponding to the reference. + /// + /// # Errors + /// + /// This function can fail only in response to external errors: a failure of the + /// `resolve_reference()` function or a failure to write to the writer. + pub fn write_to( + &self, + write: &mut impl Write, + resolve_reference: impl Fn(&ContentReference) -> Result>, + ) -> Result<()> { + for chunk in &self.chunks { + match chunk { + Chunk::Inline(data) => { + write.write_all(data)?; + } + Chunk::External(refs) => { + for r#ref in refs { + write.write_all(&resolve_reference(r#ref)?)?; + } + } + } + } + Ok(()) + } +} + +/// A reference to file metadata, either the manifest or the tarsplit +pub struct MetadataReference { + /// The range itself, in bytes, in the compressed file. + pub range: Range, + + /// The digest of the data at the range, *before* decompression. This will be missing if we + /// read from the file footer or if the OCI annotations didn't provide it. + pub digest: Option, + + /// The size of the compressed data at the range, after decompression. + pub uncompressed_size: u64, +} + +impl MetadataReference { + const fn from_footer(value: &FooterReference) -> Self { + let start = value.offset.get(); + let end = start + value.length_compressed.get(); + + Self { + range: start..end, + digest: None, + uncompressed_size: value.length_uncompressed.get(), + } + } +} + +/// References to the manifest and tarsplit metadata. You can read these from the file footer or +/// from the annotations on the OCI layer descriptor. +pub struct MetadataReferences { + /// The location of the manifest data + pub manifest: MetadataReference, + /// The location of the tarsplit data + pub tarsplit: MetadataReference, +} + +fn to_vec_u64(value: &str) -> Option> { + value.split(':').map(|s| s.parse().ok()).collect() +} + +impl MetadataReferences { + /// Read the metadata references from the file footer. The provided data can be any suffix of + /// the file, but it must be at least 72 bytes in length (to contain the footer). Returns None + /// if this doesn't appear to be a zstd:chunked file. + #[must_use] + pub fn from_footer(suffix: &[u8]) -> Option { + let footer = Footer::from_suffix(suffix)?; + Some(Self { + manifest: MetadataReference::from_footer(&footer.manifest), + tarsplit: MetadataReference::from_footer(&footer.tarsplit), + }) + } + + /// Parses the metadata references from OCI layer descriptor annotations. The type of the + /// annotations `HashMap` was chosen to be compatible with the oci-spec crate. Returns None if + /// this doesn't appear to be a zstd:chunked layer descriptor. + #[allow(clippy::needless_pass_by_value)] + #[must_use] + pub fn from_oci(annotations: HashMap) -> Option { + let manifest_digest = + annotations.get("io.github.containers.zstd-chunked.manifest-checksum"); + let manifest_position = + annotations.get("io.github.containers.zstd-chunked.manifest-position")?; + let tarsplit_digest = + annotations.get("io.github.containers.zstd-chunked.tarsplit-checksum"); + let tarsplit_position = + annotations.get("io.github.containers.zstd-chunked.tarsplit-position")?; + + Some(Self { + manifest: match to_vec_u64(manifest_position)?.as_slice() { + &[start, length, uncompressed_size, 1] => MetadataReference { + range: start..(start + length), + digest: manifest_digest.cloned(), + uncompressed_size, + }, + _ => None?, + }, + tarsplit: match to_vec_u64(tarsplit_position)?.as_slice() { + &[start, length, uncompressed_size] => MetadataReference { + range: start..(start + length), + digest: tarsplit_digest.cloned(), + uncompressed_size, + }, + _ => None?, + }, + }) + } +}