Task 644: .SCM File Format

1. Properties of the .SCM File Format Intrinsic to Its File System

The .SCM file format, as used for StarCraft maps, is fundamentally an MPQ (Mo'PaQ) archive—a container format developed by Blizzard Entertainment that functions as a self-contained file system. This structure allows multiple files to be stored within the archive, with metadata for lookup, compression, and encryption. The intrinsic properties pertain to the MPQ metadata structures (header, hash table, and block table), which define the archive's organization, file locations, sizes, and security features. These properties do not include the content of individual files (e.g., the STAREDIT\SCENARIO.CHK map data specific to StarCraft) but focus on the file system-level attributes.

The properties are derived from the MPQ specification and are as follows:

MPQ Header Properties (Offsets Relative to Archive Start)

Extended MPQ Header Properties (Present if Format Version ≥ 1; Follows Base Header)

Hash Table Entries (One per Potential File; 16 Bytes Each; Encrypted with Key "(hash table)")

Block Table Entries (One per File; 16 Bytes Each; Encrypted with Key "(block table)")

These properties collectively form the MPQ file system's directory-like structure, enabling efficient file lookup via hashes and supporting features like compression (PKWARE, Huffman) and encryption.

2. Two Direct Download Links for .SCM Files

• Micro FASTEST $$.scm (A sample micro-management map for StarCraft; ~44 KB).
• Fastest Green!!.scm (A sample fast-expansion map for StarCraft; ~44 KB).

3. Ghost Blog Embedded HTML JavaScript for Drag-and-Drop .SCM Property Dump

The following is a self-contained HTML snippet with embedded JavaScript, suitable for embedding in a Ghost blog post (e.g., via the HTML card). It enables drag-and-drop of an .SCM file in the browser, parses the MPQ structure, decrypts the tables, and dumps all intrinsic properties to the screen in a formatted <div>. It implements the MPQ parsing logic client-side using ArrayBuffer and DataView.

This code handles parsing, decryption, and display without external dependencies. Drag a .SCM file onto the zone to view the properties.

4. Python Class for .SCM Handling

The following Python class uses the struct module to parse, decrypt, and print MPQ properties from a .SCM file. It supports reading (decoding and printing) and writing (re-encoding the same data to a new file).

import struct
import os

class SCMParser:
    def __init__(self):
        self.crypt_table = self._prepare_crypt_table()

    def _prepare_crypt_table(self):
        table = [0] * 0x500
        seed = 0x00100001
        index1 = 0
        while index1 < 0x100:
            index2 = index1
            for _ in range(5):
                seed = (seed * 125 + 3) % 0x2AAAAB
                temp1 = (seed & 0xFFFF) << 16
                seed = (seed * 125 + 3) % 0x2AAAAB
                temp2 = seed & 0xFFFF
                table[index2] = temp1 | temp2
                index2 += 0x100
            index1 += 1
        return table

    def _hash_string(self, s, hash_type):
        seed1 = 0x7FED7FED
        seed2 = 0xEEEEEEEE
        for char in s.lower():
            ch = ord(char)
            seed1 = self.crypt_table[(hash_type << 8) + ch] ^ (seed1 + seed2)
            seed2 = ch + seed1 + seed2 + (seed2 << 5) + 3
        return seed1 & 0xFFFFFFFF

    def _decrypt_block(self, data, length, key):
        seed = 0xEEEEEEEE
        ch = 0
        for i in range(0, length, 4):
            seed += self.crypt_table[0x400 + (key & 0xFF)]
            ch = struct.unpack('<I', data[i:i+4])[0] ^ (key + seed)
            key = ((~key << 21) + 0x11111111) | (key >> 11)
            seed = ch + seed + (seed << 5) + 3
            data[i:i+4] = struct.pack('<I', ch)

    def read(self, filename):
        with open(filename, 'rb') as f:
            data = bytearray(f.read())
        offset = 0
        magic = struct.unpack('<I', data[offset:offset+4])[0]
        if magic != 0x1A1A514D:
            raise ValueError('Invalid MPQ magic')
        header = struct.unpack('<I I I H H I I I I', data[offset:offset+36])
        header = {
            'magic': header[0], 'header_size': header[1], 'archive_size': header[2],
            'format_version': header[3], 'block_size': header[4],
            'hash_table_pos': header[5], 'block_table_pos': header[6],
            'hash_table_size': header[7], 'block_table_size': header[8]
        }
        hash_key = self._hash_string('(hash table)', 0)
        block_key = self._hash_string('(block table)', 0)

        # Decrypt hash table
        hash_start = header['hash_table_pos']
        self._decrypt_block(data, header['hash_table_size'] * 16, hash_key)

        # Decrypt block table
        block_start = header['block_table_pos']
        self._decrypt_block(data, header['block_table_size'] * 16, block_key)

        print('MPQ Header:')
        for k, v in header.items():
            print(f'  {k}: {v}')

        if header['format_version'] >= 1:
            ext_offset = offset + header['header_size']
            ext = struct.unpack('<Q H H', data[ext_offset:ext_offset+12])
            print('Extended Header:')
            print(f'  ext_block_table_pos: {ext[0]}')
            print(f'  hash_table_pos_high: {ext[1]}')
            print(f'  block_table_pos_high: {ext[2]}')

        print('\nHash Table Entries:')
        for i in range(header['hash_table_size']):
            off = hash_start + i * 16
            entry = struct.unpack('<I I H H I', data[off:off+16])
            if entry[4] != 0xFFFFFFFF:
                print(f'  Entry {i}: name_hash1={entry[0]}, name_hash2={entry[1]}, locale={entry[2]}, platform={entry[3]}, block_index={entry[4]}')

        print('\nBlock Table Entries:')
        for i in range(header['block_table_size']):
            off = block_start + i * 16
            entry = struct.unpack('<I I I I', data[off:off+16])
            print(f'  Entry {i}: file_position={entry[0]}, compressed_size={entry[1]}, uncompressed_size={entry[2]}, flags={entry[3]:x}')

        return data  # Return modified data for write

    def write(self, data, output_filename):
        with open(output_filename, 'wb') as f:
            f.write(data)

# Usage example:
# parser = SCMParser()
# data = parser.read('example.scm')
# parser.write(data, 'output.scm')

To use: Instantiate SCMParser(), call read(filename) to decode and print properties, and write(data, output_filename) to save.

5. Java Class for .SCM Handling

The following Java class uses ByteBuffer for binary parsing. Compile with javac SCMParser.java and run with java SCMParser example.scm output.scm. It reads, prints properties, and writes back.

import java.io.*;
import java.nio.*;
import java.nio.channels.FileChannel;
import java.nio.file.*;

public class SCMParser {
    private int[] cryptTable = new int[0x500];

    public SCMParser() {
        prepareCryptTable();
    }

    private void prepareCryptTable() {
        int seed = 0x00100001, index1 = 0, index2 = 0;
        for (index1 = 0; index1 < 0x100; index1++) {
            index2 = index1;
            for (int i = 0; i < 5; i++, index2 += 0x100) {
                seed = (seed * 125 + 3) % 0x2AAAAB;
                int temp1 = (seed & 0xFFFF) << 16;
                seed = (seed * 125 + 3) % 0x2AAAAB;
                int temp2 = seed & 0xFFFF;
                cryptTable[index2] = temp1 | temp2;
            }
        }
    }

    private int hashString(String s, int hashType) {
        int seed1 = 0x7FED7FED, seed2 = 0xEEEEEEEE;
        for (char c : s.toLowerCase().toCharArray()) {
            int ch = (int) c;
            seed1 = cryptTable[(hashType << 8) + ch] ^ (seed1 + seed2);
            seed2 = ch + seed1 + seed2 + (seed2 << 5) + 3;
        }
        return seed1 & 0xFFFFFFFF;
    }

    private void decryptBlock(ByteBuffer data, int length, int key) {
        int seed = 0xEEEEEEEE, ch;
        data.position(0);
        for (int i = 0; i < length; i += 4) {
            seed += cryptTable[0x400 + (key & 0xFF)];
            ch = data.getInt() ^ (key + seed);
            key = ((~key << 21) + 0x11111111) | (key >>> 11);
            seed = ch + seed + (seed << 5) + 3;
            data.position(i);
            data.putInt(ch);
            data.position(0);
        }
    }

    public ByteBuffer read(String inputFile) throws IOException {
        Path path = Paths.get(inputFile);
        ByteBuffer data = ByteBuffer.allocateDirect((int) Files.size(path));
        try (FileChannel channel = FileChannel.open(path)) {
            channel.read(data);
        }
        data.flip();

        int offset = 0;
        if (data.getInt(offset) != 0x1A1A514D) throw new RuntimeException("Invalid MPQ magic");
        data.position(offset);
        ByteBuffer headerBuf = data.slice();
        headerBuf.limit(32);
        ByteBuffer dup = headerBuf.duplicate();
        int magic = dup.getInt();
        int headerSize = dup.getInt();
        int archiveSize = dup.getInt();
        short formatVersion = dup.getShort();
        short blockSize = dup.getShort();
        int hashTablePos = dup.getInt();
        int blockTablePos = dup.getInt();
        int hashTableSize = dup.getInt();
        int blockTableSize = dup.getInt();

        System.out.println("MPQ Header:");
        System.out.println("  magic: " + magic);
        System.out.println("  header_size: " + headerSize);
        System.out.println("  archive_size: " + archiveSize);
        System.out.println("  format_version: " + formatVersion);
        System.out.println("  block_size: " + blockSize);
        System.out.println("  hash_table_pos: " + hashTablePos);
        System.out.println("  block_table_pos: " + blockTablePos);
        System.out.println("  hash_table_size: " + hashTableSize);
        System.out.println("  block_table_size: " + blockTableSize);

        int hashKey = hashString("(hash table)", 0);
        int blockKey = hashString("(block table)", 0);

        // Decrypt hash table
        data.position(hashTablePos);
        ByteBuffer hashSlice = data.slice();
        hashSlice.limit(hashTableSize * 16);
        decryptBlock(hashSlice, hashTableSize * 16, hashKey);

        // Decrypt block table
        data.position(blockTablePos);
        ByteBuffer blockSlice = data.slice();
        blockSlice.limit(blockTableSize * 16);
        decryptBlock(blockSlice, blockTableSize * 16, blockKey);

        System.out.println("\nHash Table Entries:");
        data.position(hashTablePos);
        for (int i = 0; i < hashTableSize; i++) {
            int nameHash1 = data.getInt();
            int nameHash2 = data.getInt();
            short locale = data.getShort();
            short platform = data.getShort();
            int blockIndex = data.getInt();
            if (blockIndex != 0xFFFFFFFF) {
                System.out.printf("  Entry %d: name_hash1=%d, name_hash2=%d, locale=%d, platform=%d, block_index=%d%n",
                    i, nameHash1, nameHash2, locale, platform, blockIndex);
            }
        }

        System.out.println("\nBlock Table Entries:");
        data.position(blockTablePos);
        for (int i = 0; i < blockTableSize; i++) {
            int filePos = data.getInt();
            int compSize = data.getInt();
            int uncompSize = data.getInt();
            int flags = data.getInt();
            System.out.printf("  Entry %d: file_position=%d, compressed_size=%d, uncompressed_size=%d, flags=0x%X%n",
                i, filePos, compSize, uncompSize, flags);
        }

        if (formatVersion >= 1) {
            int extOffset = offset + headerSize;
            data.position(extOffset);
            long extBlockPos = data.getLong();
            short hashPosHigh = data.getShort();
            short blockPosHigh = data.getShort();
            System.out.println("Extended Header:");
            System.out.println("  ext_block_table_pos: " + extBlockPos);
            System.out.println("  hash_table_pos_high: " + hashPosHigh);
            System.out.println("  block_table_pos_high: " + blockPosHigh);
        }

        return data;
    }

    public void write(ByteBuffer data, String outputFile) throws IOException {
        data.rewind();
        try (FileChannel channel = FileChannel.open(Paths.get(outputFile), StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
            channel.write(data);
        }
    }

    public static void main(String[] args) throws IOException {
        if (args.length < 1) {
            System.err.println("Usage: java SCMParser <input.scm> [output.scm]");
            return;
        }
        SCMParser parser = new SCMParser();
        ByteBuffer data = parser.read(args[0]);
        if (args.length > 1) {
            parser.write(data, args[1]);
            System.out.println("Written to " + args[1]);
        }
    }
}

6. JavaScript Class for .SCM Handling (Node.js)

This Node.js class uses fs for file I/O. Run with node scm_parser.js example.scm [output.scm]. It mirrors the Python/Java functionality.

const fs = require('fs');

class SCMParser {
  constructor() {
    this.cryptTable = this.prepareCryptTable();
  }

  prepareCryptTable() {
    const table = new Uint32Array(0x500);
    let seed = 0x00100001, index1 = 0;
    while (index1 < 0x100) {
      let index2 = index1;
      for (let i = 0; i < 5; i++, index2 += 0x100) {
        seed = (seed * 125 + 3) % 0x2AAAAB;
        let temp1 = (seed & 0xFFFF) << 16;
        seed = (seed * 125 + 3) % 0x2AAAAB;
        let temp2 = seed & 0xFFFF;
        table[index2] = temp1 | temp2;
      }
      index1++;
    }
    return table;
  }

  hashString(s, hashType) {
    let seed1 = 0x7FED7FED, seed2 = 0xEEEEEEEE;
    for (let i = 0; i < s.length; i++) {
      let ch = s.charCodeAt(i);
      if (ch >= 65 && ch <= 90) ch += 32;
      seed1 = this.cryptTable[(hashType << 8) + ch] ^ (seed1 + seed2);
      seed2 = ch + seed1 + seed2 + (seed2 << 5) + 3;
    }
    return seed1 >>> 0;
  }

  decryptBlock(buffer, offset, length, key) {
    let seed = 0xEEEEEEEE;
    for (let i = 0; i < length; i += 4) {
      seed += this.cryptTable[0x400 + (key & 0xFF)];
      let ch = buffer.readUInt32LE(offset + i) ^ (key + seed);
      key = ((~key << 21) + 0x11111111) | (key >>> 11);
      seed = ch + seed + (seed << 5) + 3;
      buffer.writeUInt32LE(ch, offset + i);
    }
  }

  read(filename) {
    const data = Buffer.allocUnsafe(fs.statSync(filename).size);
    fs.readFileSync(filename).copy(data);
    let offset = 0;
    if (data.readUInt32LE(offset) !== 0x1A1A514D) throw new Error('Invalid MPQ magic');
    const header = {
      magic: data.readUInt32LE(offset),
      headerSize: data.readUInt32LE(offset + 4),
      archiveSize: data.readUInt32LE(offset + 8),
      formatVersion: data.readUInt16LE(offset + 12),
      blockSize: data.readUInt16LE(offset + 14),
      hashTablePos: data.readUInt32LE(offset + 16),
      blockTablePos: data.readUInt32LE(offset + 20),
      hashTableSize: data.readUInt32LE(offset + 24),
      blockTableSize: data.readUInt32LE(offset + 28)
    };
    console.log('MPQ Header:');
    Object.entries(header).forEach(([k, v]) => console.log(`  ${k}: ${v}`));

    const hashKey = this.hashString('(hash table)', 0);
    const blockKey = this.hashString('(block table)', 0);

    this.decryptBlock(data, header.hashTablePos, header.hashTableSize * 16, hashKey);
    this.decryptBlock(data, header.blockTablePos, header.blockTableSize * 16, blockKey);

    console.log('\nHash Table Entries:');
    for (let i = 0; i < header.hashTableSize; i++) {
      const off = header.hashTablePos + i * 16;
      const nameHash1 = data.readUInt32LE(off);
      const nameHash2 = data.readUInt32LE(off + 4);
      const locale = data.readUInt16LE(off + 8);
      const platform = data.readUInt16LE(off + 10);
      const blockIndex = data.readUInt32LE(off + 12);
      if (blockIndex !== 0xFFFFFFFF) {
        console.log(`  Entry ${i}: name_hash1=${nameHash1}, name_hash2=${nameHash2}, locale=${locale}, platform=${platform}, block_index=${blockIndex}`);
      }
    }

    console.log('\nBlock Table Entries:');
    for (let i = 0; i < header.blockTableSize; i++) {
      const off = header.blockTablePos + i * 16;
      const filePos = data.readUInt32LE(off);
      const compSize = data.readUInt32LE(off + 4);
      const uncompSize = data.readUInt32LE(off + 8);
      const flags = data.readUInt32LE(off + 12);
      console.log(`  Entry ${i}: file_position=${filePos}, compressed_size=${compSize}, uncompressed_size=${uncompSize}, flags=0x${flags.toString(16)}`);
    }

    if (header.formatVersion >= 1) {
      const extOffset = offset + header.headerSize;
      const extBlockPos = data.readBigUInt64LE(extOffset);
      const hashPosHigh = data.readUInt16LE(extOffset + 8);
      const blockPosHigh = data.readUInt16LE(extOffset + 10);
      console.log('Extended Header:');
      console.log(`  ext_block_table_pos: ${extBlockPos}`);
      console.log(`  hash_table_pos_high: ${hashPosHigh}`);
      console.log(`  block_table_pos_high: ${blockPosHigh}`);
    }

    return data;
  }

  write(data, outputFilename) {
    fs.writeFileSync(outputFilename, data);
  }
}

// Usage: node scm_parser.js example.scm [output.scm]
const args = process.argv.slice(2);
if (args.length < 1) {
  console.error('Usage: node scm_parser.js <input.scm> [output.scm]');
  process.exit(1);
}
const parser = new SCMParser();
const data = parser.read(args[0]);
if (args[1]) {
  parser.write(data, args[1]);
  console.log(`Written to ${args[1]}`);
}

7. C Class (Struct with Functions) for .SCM Handling

This C implementation uses standard I/O and manual buffer management. Compile with gcc -o scm_parser scm_parser.c and run ./scm_parser example.scm [output.scm]. It reads, prints, and writes properties.

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <ctype.h>

typedef uint32_t u32;
typedef uint16_t u16;
typedef uint64_t u64;

u32 crypt_table[0x500];

void prepare_crypt_table() {
    u32 seed = 0x00100001, index1 = 0;
    while (index1 < 0x100) {
        u32 index2 = index1;
        for (int i = 0; i < 5; i++, index2 += 0x100) {
            seed = (seed * 125 + 3) % 0x2AAAAB;
            u32 temp1 = (seed & 0xFFFF) << 16;
            seed = (seed * 125 + 3) % 0x2AAAAB;
            u32 temp2 = seed & 0xFFFF;
            crypt_table[index2] = temp1 | temp2;
        }
        index1++;
    }
}

u32 hash_string(const char *s, u32 hash_type) {
    u32 seed1 = 0x7FED7FED, seed2 = 0xEEEEEEEE;
    for (int i = 0; s[i]; i++) {
        int ch = tolower(s[i]);
        seed1 = crypt_table[(hash_type << 8) + ch] ^ (seed1 + seed2);
        seed2 = ch + seed1 + seed2 + (seed2 << 5) + 3;
    }
    return seed1;
}

void decrypt_block(uint8_t *data, size_t length, u32 key) {
    u32 seed = 0xEEEEEEEE;
    for (size_t i = 0; i < length; i += 4) {
        seed += crypt_table[0x400 + (key & 0xFF)];
        u32 ch = *(u32 *)(data + i) ^ (key + seed);
        key = ((~key << 21) + 0x11111111) | (key >> 11);
        seed = ch + seed + (seed << 5) + 3;
        *(u32 *)(data + i) = ch;
    }
}

uint8_t *read_scm(const char *filename, size_t *size) {
    FILE *f = fopen(filename, "rb");
    if (!f) return NULL;
    fseek(f, 0, SEEK_END);
    *size = ftell(f);
    fseek(f, 0, SEEK_SET);
    uint8_t *data = malloc(*size);
    fread(data, 1, *size, f);
    fclose(f);
    return data;
}

void print_properties(uint8_t *data, size_t size) {
    u32 offset = 0;
    u32 magic = *(u32 *)(data + offset);
    if (magic != 0x1A1A514D) {
        fprintf(stderr, "Invalid MPQ magic\n");
        return;
    }
    u32 header_size = *(u32 *)(data + offset + 4);
    u32 archive_size = *(u32 *)(data + offset + 8);
    u16 format_version = *(u16 *)(data + offset + 12);
    u16 block_size = *(u16 *)(data + offset + 14);
    u32 hash_table_pos = *(u32 *)(data + offset + 16);
    u32 block_table_pos = *(u32 *)(data + offset + 20);
    u32 hash_table_size = *(u32 *)(data + offset + 24);
    u32 block_table_size = *(u32 *)(data + offset + 28);

    printf("MPQ Header:\n");
    printf("  magic: %u\n", magic);
    printf("  header_size: %u\n", header_size);
    printf("  archive_size: %u\n", archive_size);
    printf("  format_version: %u\n", format_version);
    printf("  block_size: %u\n", block_size);
    printf("  hash_table_pos: %u\n", hash_table_pos);
    printf("  block_table_pos: %u\n", block_table_pos);
    printf("  hash_table_size: %u\n", hash_table_size);
    printf("  block_table_size: %u\n", block_table_size);

    u32 hash_key = hash_string("(hash table)", 0);
    u32 block_key = hash_string("(block table)", 0);

    decrypt_block(data + hash_table_pos, hash_table_size * 16, hash_key);
    decrypt_block(data + block_table_pos, block_table_size * 16, block_key);

    printf("\nHash Table Entries:\n");
    for (u32 i = 0; i < hash_table_size; i++) {
        u32 off = hash_table_pos + i * 16;
        u32 name_hash1 = *(u32 *)(data + off);
        u32 name_hash2 = *(u32 *)(data + off + 4);
        u16 locale = *(u16 *)(data + off + 8);
        u16 platform = *(u16 *)(data + off + 10);
        u32 block_index = *(u32 *)(data + off + 12);
        if (block_index != 0xFFFFFFFF) {
            printf("  Entry %u: name_hash1=%u, name_hash2=%u, locale=%u, platform=%u, block_index=%u\n",
                   i, name_hash1, name_hash2, locale, platform, block_index);
        }
    }

    printf("\nBlock Table Entries:\n");
    for (u32 i = 0; i < block_table_size; i++) {
        u32 off = block_table_pos + i * 16;
        u32 file_pos = *(u32 *)(data + off);
        u32 comp_size = *(u32 *)(data + off + 4);
        u32 uncomp_size = *(u32 *)(data + off + 8);
        u32 flags = *(u32 *)(data + off + 12);
        printf("  Entry %u: file_position=%u, compressed_size=%u, uncompressed_size=%u, flags=0x%X\n",
               i, file_pos, comp_size, uncomp_size, flags);
    }

    if (format_version >= 1) {
        u32 ext_offset = offset + header_size;
        u64 ext_block_pos = *(u64 *)(data + ext_offset);
        u16 hash_pos_high = *(u16 *)(data + ext_offset + 8);
        u16 block_pos_high = *(u16 *)(data + ext_offset + 10);
        printf("Extended Header:\n");
        printf("  ext_block_table_pos: %lu\n", ext_block_pos);
        printf("  hash_table_pos_high: %u\n", hash_pos_high);
        printf("  block_table_pos_high: %u\n", block_pos_high);
    }
}

void write_scm(const uint8_t *data, size_t size, const char *output) {
    FILE *f = fopen(output, "wb");
    if (f) {
        fwrite(data, 1, size, f);
        fclose(f);
    }
}

int main(int argc, char **argv) {
    if (argc < 2) {
        fprintf(stderr, "Usage: %s <input.scm> [output.scm]\n", argv[0]);
        return 1;
    }
    prepare_crypt_table();
    size_t size;
    uint8_t *data = read_scm(argv[1], &size);
    if (data) {
        print_properties(data, size);
        if (argc > 2) {
            write_scm(data, size, argv[2]);
            printf("Written to %s\n", argv[2]);
        }
        free(data);
    }
    return 0;
}