.SDTS

Task 651: .SDTS File Format

Task 651: .SDTS File Format

thinkmelt@protonmail.com

• 6 min read

SDTS File Format Specifications

The .SDTS file format refers to archives containing Spatial Data Transfer Standard (SDTS) data, typically bundled as .tar.gz files with multiple .DDF (ISO 8211-encoded) modules inside. SDTS is a standard for transferring geospatial vector and raster data, originally developed by the USGS and standardized under ANSI INCITS 320-1998. The core specifications are detailed in the SDTS Part 1: Logical Specifications, available as a PDF at https://rosap.ntl.bts.gov/view/dot/4529/dot_4529_DS1.pdf. Additional parts cover spatial features, encoding, and profiles (e.g., Topological Vector Profile).

List of all properties of this file format intrinsic to its file system:
These are the module types (4-letter tags) defined in the SDTS specifications, which represent the core structural components (properties) of the format. Each module encapsulates specific data or metadata, and their presence defines the dataset's organization.

  • IDEN (Identification)
  • CATD (Catalog/Directory)
  • CATX (Catalog/Cross Reference)
  • CATS (Catalog/Spatial Domain)
  • SCUR (Security)
  • IREF (Internal Spatial Reference)
  • XREF (External Spatial Reference)
  • SPDM (Spatial Domain)
  • DDDF (Data Dictionary/Definition)
  • DDOM (Data Dictionary/Domain)
  • DDSH (Data Dictionary/Schema)
  • STAT (Transfer Statistics)
  • DQHL (Data Quality/Lineage)
  • DQPA (Data Quality/Positional Accuracy)
  • DQAA (Data Quality/Attribute Accuracy)
  • DQLC (Data Quality/Logical Consistency)
  • DQCG (Data Quality/Completeness)
  • AP00/AP01/Axxx (Attribute Primary, where xxx is variable)
  • Bxxx (Attribute Secondary, where xxx is variable)
  • NO01 (Planar Node)
  • NE01 (Entity Point)
  • NA01 (Area Point)
  • LE01 (Line/Complete Chain)
  • PC01 (Polygon/GT-Polygon)
  • FFxx (Composite Object, where xx is variable)

Two direct download links for files of format .SDTS:

Ghost blog embedded HTML JavaScript for drag-and-drop .SDTS file to dump properties:

SDTS Property Dumper
Drag and drop .SDTS file here

  


Python class for .SDTS file handling:
import tarfile
import gzip
import os

class SDTSHandler:
    def __init__(self, filename):
        self.filename = filename
        self.properties = set()

    def read(self):
        # Open and extract module properties from .SDTS (tar.gz)
        with gzip.open(self.filename, 'rb') as gz:
            with tarfile.open(fileobj=gz) as tar:
                for member in tar.getmembers():
                    if member.name.endswith('.DDF'):
                        module_tag = member.name[-8:-4]  # Extract tag like 'IDEN'
                        self.properties.add(module_tag)

    def print_properties(self):
        print("Properties (module tags) found:")
        for prop in sorted(self.properties):
            print(prop)

    def write(self, output_filename, modules):
        # Write a new .SDTS file with dummy modules (for demonstration; add real content as needed)
        with tarfile.open(output_filename, 'w:gz') as tar:
            for tag in modules:
                dummy_file = f'dummy{tag}.DDF'
                with open(dummy_file, 'w') as f:
                    f.write(f'Dummy content for {tag}')
                tar.add(dummy_file)
                os.remove(dummy_file)

# Example usage:
# handler = SDTSHandler('example.sdts')
# handler.read()
# handler.print_properties()
# handler.write('new.sdts', ['IDEN', 'CATD'])

Java class for .SDTS file handling:
import java.io.*;
import java.util.*;
import java.util.zip.GZIPInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;

public class SDTSHandler {
    private String filename;
    private Set<String> properties = new HashSet<>();

    public SDTSHandler(String filename) {
        this.filename = filename;
    }

    public void read() throws IOException {
        try (FileInputStream fis = new FileInputStream(filename);
             GZIPInputStream gis = new GZIPInputStream(fis);
             TarArchiveInputStream tis = new TarArchiveInputStream(gis)) {
            TarArchiveEntry entry;
            while ((entry = tis.getNextTarEntry()) != null) {
                if (entry.getName().endsWith(".DDF")) {
                    String moduleTag = entry.getName().substring(entry.getName().length() - 8, entry.getName().length() - 4);
                    properties.add(moduleTag);
                }
            }
        }
    }

    public void printProperties() {
        System.out.println("Properties (module tags) found:");
        for (String prop : new TreeSet<>(properties)) {
            System.out.println(prop);
        }
    }

    public void write(String outputFilename, List<String> modules) throws IOException {
        try (FileOutputStream fos = new FileOutputStream(outputFilename);
             TarArchiveOutputStream tos = new TarArchiveOutputStream(new GZIPOutputStream(fos))) {
            for (String tag : modules) {
                String dummyFile = "dummy" + tag + ".DDF";
                try (FileWriter fw = new FileWriter(dummyFile)) {
                    fw.write("Dummy content for " + tag);
                }
                File file = new File(dummyFile);
                TarArchiveEntry entry = new TarArchiveEntry(file, dummyFile);
                tos.putArchiveEntry(entry);
                try (FileInputStream fis = new FileInputStream(file)) {
                    byte[] buffer = new byte[1024];
                    int len;
                    while ((len = fis.read(buffer)) > 0) {
                        tos.write(buffer, 0, len);
                    }
                }
                tos.closeArchiveEntry();
                file.delete();
            }
        }
    }

    // Example usage:
    // public static void main(String[] args) throws IOException {
    //     SDTSHandler handler = new SDTSHandler("example.sdts");
    //     handler.read();
    //     handler.printProperties();
    //     handler.write("new.sdts", Arrays.asList("IDEN", "CATD"));
    // }
}

JavaScript class for .SDTS file handling:
class SDTSHandler {
    constructor(file) {
        this.file = file;
        this.properties = new Set();
    }

    async read() {
        const arrayBuffer = await this.file.arrayBuffer();
        const uint8Array = new Uint8Array(arrayBuffer);

        // Decompress gzip
        const decompressedStream = new DecompressionStream('gzip');
        const writer = decompressedStream.writable.getWriter();
        writer.write(uint8Array);
        writer.close();
        const decompressed = await new Response(decompressedStream.readable).arrayBuffer();
        const decompressedUint8 = new Uint8Array(decompressed);

        // Parse TAR
        let pos = 0;
        while (pos < decompressedUint8.length) {
            const header = decompressedUint8.subarray(pos, pos + 512);
            if (header.every(v => v === 0)) break;
            const name = new TextDecoder().decode(header.subarray(0, 100)).replace(/\0/g, '');
            if (name.endsWith('.DDF')) {
                const moduleTag = name.slice(-8, -4);
                this.properties.add(moduleTag);
            }
            const sizeOct = parseInt(new TextDecoder().decode(header.subarray(124, 136)).replace(/\0/g, ''), 8);
            pos += 512 + Math.ceil(sizeOct / 512) * 512;
        }
    }

    printProperties() {
        console.log('Properties (module tags) found:');
        [...this.properties].sort().forEach(prop => console.log(prop));
    }

    async write(outputFilename, modules) {
        // Writing in browser JS is limited; this simulates by creating a Blob (download manually)
        const tarContent = [];
        modules.forEach(tag => {
            const dummyContent = `Dummy content for ${tag}`;
            const dummyName = `dummy${tag}.DDF`;
            const header = new Uint8Array(512).fill(0);
            new TextEncoder().encode(dummyName).forEach((b, i) => header[i] = b);
            const sizeStr = dummyContent.length.toString(8).padStart(11, '0') + '\0';
            new TextEncoder().encode(sizeStr).forEach((b, i) => header[124 + i] = b);
            tarContent.push(header);
            tarContent.push(new TextEncoder().encode(dummyContent));
            // Pad to 512
            const pad = new Uint8Array(512 - (dummyContent.length % 512)).fill(0);
            if (pad.length < 512) tarContent.push(pad);
        });
        tarContent.push(new Uint8Array(1024).fill(0)); // End of TAR
        const tarBlob = new Blob(tarContent);

        // Compress to gzip using CompressionStream
        const compressedStream = tarBlob.stream().pipeThrough(new CompressionStream('gzip'));
        const compressedBlob = await new Response(compressedStream).blob();

        // Download
        const a = document.createElement('a');
        a.href = URL.createObjectURL(compressedBlob);
        a.download = outputFilename;
        a.click();
    }
}

// Example usage:
// const file = ... // from input or drop
// const handler = new SDTSHandler(file);
// await handler.read();
// handler.printProperties();
// await handler.write('new.sdts', ['IDEN', 'CATD']);

C++ class for .SDTS file handling (assuming zlib for gzip and no external tar lib; simple tar parser implemented):
#include <iostream>
#include <fstream>
#include <set>
#include <string>
#include <vector>
#include <zlib.h>  // Assume zlib linked for gzip

class SDTSHandler {
private:
    std::string filename;
    std::set<std::string> properties;

    std::vector<char> decompressGzip(const std::vector<char>& data) {
        z_stream zs = {};
        inflateInit2(&zs, 16 + MAX_WBITS);  // Gzip
        zs.next_in = reinterpret_cast<Bytef*>(const_cast<char*>(data.data()));
        zs.avail_in = data.size();
        std::vector<char> out;
        char buffer[1024];
        do {
            zs.next_out = reinterpret_cast<Bytef*>(buffer);
            zs.avail_out = sizeof(buffer);
            inflate(&zs, Z_NO_FLUSH);
            out.insert(out.end(), buffer, buffer + sizeof(buffer) - zs.avail_out);
        } while (zs.avail_out == 0);
        inflateEnd(&zs);
        return out;
    }

public:
    SDTSHandler(const std::string& fn) : filename(fn) {}

    void read() {
        std::ifstream file(filename, std::ios::binary);
        std::vector<char> compressed((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
        auto decompressed = decompressGzip(compressed);

        // Parse TAR
        size_t pos = 0;
        while (pos < decompressed.size()) {
            const char* header = &decompressed[pos];
            if (std::string(header, 512) == std::string(512, '\0')) break;
            std::string name(header, 100);
            name = name.substr(0, name.find('\0'));
            if (name.rfind(".DDF") != std::string::npos) {
                std::string moduleTag = name.substr(name.length() - 8, 4);
                properties.insert(moduleTag);
            }
            long sizeOct = std::stol(std::string(header + 124, 12), nullptr, 8);
            pos += 512 + ((sizeOct + 511) / 512) * 512;
        }
    }

    void printProperties() {
        std::cout << "Properties (module tags) found:" << std::endl;
        for (const auto& prop : properties) {
            std::cout << prop << std::endl;
        }
    }

    void write(const std::string& outputFilename, const std::vector<std::string>& modules) {
        std::ofstream tarFile("temp.tar", std::ios::binary);
        for (const auto& tag : modules) {
            std::string dummyName = "dummy" + tag + ".DDF";
            std::string dummyContent = "Dummy content for " + tag;
            std::ofstream dummy(dummyName);
            dummy << dummyContent;
            dummy.close();

            char header[512] = {};
            std::strcpy(header, dummyName.c_str());
            std::sprintf(header + 124, "%011lo", dummyContent.length());
            tarFile.write(header, 512);
            std::ifstream dummyIn(dummyName, std::ios::binary);
            tarFile << dummyIn.rdbuf();
            size_t pad = 512 - (dummyContent.length() % 512);
            if (pad < 512) tarFile.write(std::string(pad, '\0').c_str(), pad);
            std::remove(dummyName.c_str());
        }
        tarFile.write(std::string(1024, '\0').c_str(), 1024);
        tarFile.close();

        // Compress to gzip
        gzFile gz = gzopen(outputFilename.c_str(), "wb");
        std::ifstream tarIn("temp.tar", std::ios::binary);
        char buffer[1024];
        while (tarIn.read(buffer, sizeof(buffer))) {
            gzwrite(gz, buffer, tarIn.gcount());
        }
        gzclose(gz);
        std::remove("temp.tar");
    }
};

// Example usage:
// int main() {
//     SDTSHandler handler("example.sdts");
//     handler.read();
//     handler.printProperties();
//     handler.write("new.sdts", {"IDEN", "CATD"});
//     return 0;
// }