Task 752: .UPS File Format
Task 752: .UPS File Format
UPS File Format Specifications
The .UPS file format is a binary patch format designed for applying modifications to files, particularly ROM images in emulation and hacking contexts. It supports arbitrary file sizes, bi-directional patching, and integrity verification via CRC32 checksums. The format uses variable-length integers (VLEs) for efficient encoding of large values. The VLE encoding is a variant where the high bit indicates the last byte (set for stop, clear for continue), and bits are assembled in little-endian order from 7-bit chunks.
1. List of Properties Intrinsic to the File Format
The following are the core structural properties of the .UPS file format:
- Magic Number: A 4-byte ASCII string "UPS1" identifying the file type.
- Source File Size: A variable-length integer representing the size of the original (unpatched) file in bytes.
- Target File Size: A variable-length integer representing the size of the resulting (patched) file in bytes.
- Patch Hunks: A sequence of variable-length hunks describing modifications. Each hunk consists of:
- Relative Offset: A variable-length integer indicating the number of bytes to skip (copy unchanged) from the previous position.
- XOR Data Block: A sequence of bytes to XOR with the source file, terminated by and including a zero byte (0x00). The zero byte is applied and advances the file pointer.
- Source CRC32 Checksum: A 4-byte little-endian unsigned integer for the CRC32 of the original file.
- Target CRC32 Checksum: A 4-byte little-endian unsigned integer for the CRC32 of the patched file.
- Patch CRC32 Checksum: A 4-byte little-endian unsigned integer for the CRC32 of the patch file content excluding this final checksum.
2. Two Direct Download Links for .UPS Files
- https://raw.githubusercontent.com/Veslyquix/SRR_FEGBA/main/HackInstallers/srr_ProphecyOfFlames.ups
- https://raw.githubusercontent.com/Veslyquix/SRR_FEGBA/main/HackInstallers/srr_SteelWill.ups
3. Ghost Blog Embedded HTML JavaScript for Drag-and-Drop .UPS File Analysis
The following is a self-contained HTML page with embedded JavaScript that enables drag-and-drop functionality for a .UPS file. Upon dropping the file, it parses the content and displays all properties listed in section 1 on the screen.
Drag and drop a .UPS file here
4. Python Class for .UPS File Handling
The following Python class can open a .UPS file, decode its properties, print them to the console, and write a new .UPS file based on provided properties.
import struct
import zlib
class UPSFile:
def __init__(self):
self.magic = b'UPS1'
self.source_size = 0
self.target_size = 0
self.hunks = [] # List of tuples: (offset, xor_data_bytes including terminating 0)
self.source_crc = 0
self.target_crc = 0
self.patch_crc = 0
def read_vle(self, data, pos):
value = 0
shift = 0
while True:
byte = data[pos]
pos += 1
if byte & 0x80:
value += (byte & 0x7F) << shift
break
value += (byte | 0x80) << shift
shift += 7
return value, pos
def open_and_decode(self, filename):
with open(filename, 'rb') as f:
data = f.read()
pos = 0
self.magic = data[pos:pos+4]
pos += 4
if self.magic != b'UPS1':
raise ValueError("Invalid UPS file")
self.source_size, pos = self.read_vle(data, pos)
self.target_size, pos = self.read_vle(data, pos)
self.hunks = []
end_pos = len(data) - 12
while pos < end_pos:
offset, pos = self.read_vle(data, pos)
xor_data = bytearray()
while pos < end_pos:
byte = data[pos]
pos += 1
xor_data.append(byte)
if byte == 0:
break
self.hunks.append((offset, bytes(xor_data)))
self.source_crc = struct.unpack('<I', data[pos:pos+4])[0]
self.target_crc = struct.unpack('<I', data[pos+4:pos+8])[0]
self.patch_crc = struct.unpack('<I', data[pos+8:pos+12])[0]
def print_properties(self):
print(f"Magic Number: {self.magic.decode()}")
print(f"Source File Size: {self.source_size}")
print(f"Target File Size: {self.target_size}")
print("Patch Hunks:")
for i, (offset, xor_data) in enumerate(self.hunks):
print(f" Hunk {i}:")
print(f" Relative Offset: {offset}")
print(f" XOR Data (hex): {' '.join(f'{b:02x}' for b in xor_data)}")
print(f"Source CRC32: 0x{self.source_crc:08X}")
print(f"Target CRC32: 0x{self.target_crc:08X}")
print(f"Patch CRC32: 0x{self.patch_crc:08X}")
def write_vle(self, value):
data = bytearray()
while True:
byte = value & 0x7F
value >>= 7
if value == 0:
data.append(byte | 0x80)
break
data.append(byte)
value -= 1 # Adjust for the encoding
return data
def write(self, filename):
data = bytearray(self.magic)
data += self.write_vle(self.source_size)
data += self.write_vle(self.target_size)
for offset, xor_data in self.hunks:
data += self.write_vle(offset)
data += xor_data
data += struct.pack('<I', self.source_crc)
data += struct.pack('<I', self.target_crc)
# Compute patch CRC excluding the last 4 bytes
temp_data = data + b'\x00\x00\x00\x00' # Placeholder
self.patch_crc = zlib.crc32(temp_data[:-4])
data += struct.pack('<I', self.patch_crc)
with open(filename, 'wb') as f:
f.write(data)
5. Java Class for .UPS File Handling
The following Java class can open a .UPS file, decode its properties, print them to the console, and write a new .UPS file based on provided properties.
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.CRC32;
public class UPSFile {
private String magic = "UPS1";
private long sourceSize = 0;
private long targetSize = 0;
private List<Hunk> hunks = new ArrayList<>();
private int sourceCRC = 0;
private int targetCRC = 0;
private int patchCRC = 0;
static class Hunk {
long offset;
byte[] xorData; // Including terminating 0
Hunk(long offset, byte[] xorData) {
this.offset = offset;
this.xorData = xorData;
}
}
private long readVLE(DataInputStream dis) throws IOException {
long value = 0;
int shift = 0;
while (true) {
int byteVal = dis.readUnsignedByte();
if ((byteVal & 0x80) != 0) {
value += (long) (byteVal & 0x7F) << shift;
break;
}
value += (long) (byteVal | 0x80) << shift;
shift += 7;
}
return value;
}
public void openAndDecode(String filename) throws IOException {
try (FileInputStream fis = new FileInputStream(filename);
DataInputStream dis = new DataInputStream(fis)) {
byte[] magicBytes = new byte[4];
dis.readFully(magicBytes);
magic = new String(magicBytes);
if (!magic.equals("UPS1")) {
throw new IOException("Invalid UPS file");
}
sourceSize = readVLE(dis);
targetSize = readVLE(dis);
hunks.clear();
ByteArrayOutputStream allData = new ByteArrayOutputStream();
while (fis.available() > 12) {
long offset = readVLE(dis);
ByteArrayOutputStream xorStream = new ByteArrayOutputStream();
int byteVal;
do {
byteVal = dis.readUnsignedByte();
xorStream.write(byteVal);
} while (byteVal != 0 && fis.available() > 12);
hunks.add(new Hunk(offset, xorStream.toByteArray()));
}
sourceCRC = Integer.reverseBytes(dis.readInt());
targetCRC = Integer.reverseBytes(dis.readInt());
patchCRC = Integer.reverseBytes(dis.readInt());
}
}
public void printProperties() {
System.out.println("Magic Number: " + magic);
System.out.println("Source File Size: " + sourceSize);
System.out.println("Target File Size: " + targetSize);
System.out.println("Patch Hunks:");
for (int i = 0; i < hunks.size(); i++) {
Hunk hunk = hunks.get(i);
System.out.println(" Hunk " + i + ":");
System.out.println(" Relative Offset: " + hunk.offset);
System.out.print(" XOR Data (hex): ");
for (byte b : hunk.xorData) {
System.out.printf("%02X ", b);
}
System.out.println();
}
System.out.printf("Source CRC32: 0x%08X\n", sourceCRC);
System.out.printf("Target CRC32: 0x%08X\n", targetCRC);
System.out.printf("Patch CRC32: 0x%08X\n", patchCRC);
}
private byte[] writeVLE(long value) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
while (true) {
int byteVal = (int) (value & 0x7F);
value >>= 7;
if (value == 0) {
baos.write(byteVal | 0x80);
break;
}
baos.write(byteVal);
value--;
}
return baos.toByteArray();
}
public void write(String filename) throws IOException {
try (FileOutputStream fos = new FileOutputStream(filename);
DataOutputStream dos = new DataOutputStream(fos)) {
dos.writeBytes(magic);
dos.write(writeVLE(sourceSize));
dos.write(writeVLE(targetSize));
for (Hunk hunk : hunks) {
dos.write(writeVLE(hunk.offset));
dos.write(hunk.xorData);
}
dos.writeInt(Integer.reverseBytes(sourceCRC));
dos.writeInt(Integer.reverseBytes(targetCRC));
// Compute patch CRC
ByteArrayOutputStream temp = new ByteArrayOutputStream();
DataOutputStream tempDos = new DataOutputStream(temp);
tempDos.writeBytes(magic);
tempDos.write(writeVLE(sourceSize));
tempDos.write(writeVLE(targetSize));
for (Hunk hunk : hunks) {
tempDos.write(writeVLE(hunk.offset));
tempDos.write(hunk.xorData);
}
tempDos.writeInt(Integer.reverseBytes(sourceCRC));
tempDos.writeInt(Integer.reverseBytes(targetCRC));
byte[] tempData = temp.toByteArray();
CRC32 crc = new CRC32();
crc.update(tempData);
patchCRC = (int) crc.getValue();
dos.writeInt(Integer.reverseBytes(patchCRC));
}
}
}
6. JavaScript Class for .UPS File Handling
The following JavaScript class can open a .UPS file (using Node.js fs module), decode its properties, print them to the console, and write a new .UPS file based on provided properties.
const fs = require('fs');
const crc32 = require('crc').crc32; // Requires 'crc' package for CRC32
class UPSFile {
constructor() {
this.magic = 'UPS1';
this.sourceSize = 0;
this.targetSize = 0;
this.hunks = []; // Array of {offset, xorData (Buffer including 0)}
this.sourceCRC = 0;
this.targetCRC = 0;
this.patchCRC = 0;
}
readVLE(buffer, pos) {
let value = 0n;
let shift = 0n;
while (true) {
const byte = buffer[pos++];
if (byte & 0x80) {
value += BigInt(byte & 0x7F) << shift;
break;
}
value += BigInt(byte | 0x80) << shift;
shift += 7n;
}
return { value: Number(value), pos };
}
openAndDecode(filename) {
const data = fs.readFileSync(filename);
let pos = 0;
this.magic = data.toString('ascii', pos, pos + 4);
pos += 4;
if (this.magic !== 'UPS1') {
throw new Error('Invalid UPS file');
}
let res = this.readVLE(data, pos);
this.sourceSize = res.value;
pos = res.pos;
res = this.readVLE(data, pos);
this.targetSize = res.value;
pos = res.pos;
this.hunks = [];
const endPos = data.length - 12;
while (pos < endPos) {
res = this.readVLE(data, pos);
const offset = res.value;
pos = res.pos;
const xorData = [];
let byte;
do {
byte = data[pos++];
xorData.push(byte);
} while (byte !== 0 && pos < endPos);
this.hunks.push({ offset, xorData: Buffer.from(xorData) });
}
this.sourceCRC = data.readUInt32LE(pos);
this.targetCRC = data.readUInt32LE(pos + 4);
this.patchCRC = data.readUInt32LE(pos + 8);
}
printProperties() {
console.log(`Magic Number: ${this.magic}`);
console.log(`Source File Size: ${this.sourceSize}`);
console.log(`Target File Size: ${this.targetSize}`);
console.log('Patch Hunks:');
this.hunks.forEach((hunk, i) => {
console.log(` Hunk ${i}:`);
console.log(` Relative Offset: ${hunk.offset}`);
console.log(` XOR Data (hex): ${hunk.xorData.toString('hex').match(/.{1,2}/g).join(' ')}`);
});
console.log(`Source CRC32: 0x${this.sourceCRC.toString(16).toUpperCase()}`);
console.log(`Target CRC32: 0x${this.targetCRC.toString(16).toUpperCase()}`);
console.log(`Patch CRC32: 0x${this.patchCRC.toString(16).toUpperCase()}`);
}
writeVLE(value) {
const data = [];
while (true) {
let byte = value & 0x7F;
value = Math.floor(value / 128);
if (value === 0) {
data.push(byte | 0x80);
break;
}
data.push(byte);
value--;
}
return Buffer.from(data);
}
write(filename) {
let data = Buffer.from(this.magic, 'ascii');
data = Buffer.concat([data, this.writeVLE(this.sourceSize)]);
data = Buffer.concat([data, this.writeVLE(this.targetSize)]);
this.hunks.forEach(hunk => {
data = Buffer.concat([data, this.writeVLE(hunk.offset)]);
data = Buffer.concat([data, hunk.xorData]);
});
const crcBuffer = Buffer.alloc(4);
crcBuffer.writeUInt32LE(this.sourceCRC, 0);
data = Buffer.concat([data, crcBuffer]);
crcBuffer.writeUInt32LE(this.targetCRC, 0);
data = Buffer.concat([data, crcBuffer]);
// Compute patch CRC
this.patchCRC = crc32(data);
crcBuffer.writeUInt32LE(this.patchCRC, 0);
data = Buffer.concat([data, crcBuffer]);
fs.writeFileSync(filename, data);
}
}
7. C++ Class for .UPS File Handling
The following C++ class can open a .UPS file, decode its properties, print them to the console, and write a new .UPS file based on provided properties.
#include <fstream>
#include <vector>
#include <iostream>
#include <iomanip>
#include <zlib.h> // For CRC32
struct Hunk {
uint64_t offset;
std::vector<uint8_t> xorData; // Including terminating 0
};
class UPSFile {
private:
std::string magic = "UPS1";
uint64_t sourceSize = 0;
uint64_t targetSize = 0;
std::vector<Hunk> hunks;
uint32_t sourceCRC = 0;
uint32_t targetCRC = 0;
uint32_t patchCRC = 0;
uint64_t readVLE(std::ifstream& file) {
uint64_t value = 0;
int shift = 0;
while (true) {
uint8_t byte;
file.read(reinterpret_cast<char*>(&byte), 1);
if (byte & 0x80) {
value += static_cast<uint64_t>(byte & 0x7F) << shift;
break;
}
value += static_cast<uint64_t>(byte | 0x80) << shift;
shift += 7;
}
return value;
}
void writeVLE(std::ofstream& file, uint64_t value) {
while (true) {
uint8_t byte = static_cast<uint8_t>(value & 0x7F);
value >>= 7;
if (value == 0) {
file.write(reinterpret_cast<char*>(& (uint8_t)(byte | 0x80)), 1);
break;
}
file.write(reinterpret_cast<char*>(&byte), 1);
value--;
}
}
public:
void openAndDecode(const std::string& filename) {
std::ifstream file(filename, std::ios::binary);
if (!file) {
throw std::runtime_error("Cannot open file");
}
char magicBuf[5] = {0};
file.read(magicBuf, 4);
magic = magicBuf;
if (magic != "UPS1") {
throw std::runtime_error("Invalid UPS file");
}
sourceSize = readVLE(file);
targetSize = readVLE(file);
hunks.clear();
file.seekg(0, std::ios::end);
size_t fileSize = file.tellg();
file.seekg(file.tellg() - (std::streampos)12, std::ios::beg); // Position before CRCs
size_t endPos = file.tellg();
file.seekg(4 + (magic.size() + sizeof(sourceSize) + sizeof(targetSize)), std::ios::beg); // Approximate, but reset properly
file.seekg(4, std::ios::beg); // Reset to after magic
sourceSize = readVLE(file);
targetSize = readVLE(file);
size_t pos = file.tellg();
while (pos < endPos) {
uint64_t offset = readVLE(file);
std::vector<uint8_t> xorData;
uint8_t byte;
do {
file.read(reinterpret_cast<char*>(&byte), 1);
xorData.push_back(byte);
pos = file.tellg();
} while (byte != 0 && pos < endPos);
hunks.push_back({offset, xorData});
}
file.read(reinterpret_cast<char*>(&sourceCRC), 4);
file.read(reinterpret_cast<char*>(&targetCRC), 4);
file.read(reinterpret_cast<char*>(&patchCRC), 4);
}
void printProperties() const {
std::cout << "Magic Number: " << magic << std::endl;
std::cout << "Source File Size: " << sourceSize << std::endl;
std::cout << "Target File Size: " << targetSize << std::endl;
std::cout << "Patch Hunks:" << std::endl;
for (size_t i = 0; i < hunks.size(); ++i) {
const auto& hunk = hunks[i];
std::cout << " Hunk " << i << ":" << std::endl;
std::cout << " Relative Offset: " << hunk.offset << std::endl;
std::cout << " XOR Data (hex): ";
for (uint8_t b : hunk.xorData) {
std::cout << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(b) << " ";
}
std::cout << std::dec << std::endl;
}
std::cout << "Source CRC32: 0x" << std::hex << std::uppercase << std::setw(8) << std::setfill('0') << sourceCRC << std::endl;
std::cout << "Target CRC32: 0x" << std::hex << std::uppercase << std::setw(8) << std::setfill('0') << targetCRC << std::endl;
std::cout << "Patch CRC32: 0x" << std::hex << std::uppercase << std::setw(8) << std::setfill('0') << patchCRC << std::dec << std::endl;
}
void write(const std::string& filename) {
std::ofstream file(filename, std::ios::binary);
if (!file) {
throw std::runtime_error("Cannot open file for writing");
}
file.write(magic.c_str(), 4);
writeVLE(file, sourceSize);
writeVLE(file, targetSize);
for (const auto& hunk : hunks) {
writeVLE(file, hunk.offset);
file.write(reinterpret_cast<const char*>(hunk.xorData.data()), hunk.xorData.size());
}
file.write(reinterpret_cast<const char*>(&sourceCRC), 4);
file.write(reinterpret_cast<const char*>(&targetCRC), 4);
// Compute patch CRC
file.seekp(0, std::ios::end);
size_t sizeWithoutCRC = file.tellp();
std::vector<uint8_t> tempData(sizeWithoutCRC);
file.seekp(0, std::ios::beg);
file.read(reinterpret_cast<char*>(tempData.data()), sizeWithoutCRC);
patchCRC = crc32(0L, tempData.data(), sizeWithoutCRC);
file.write(reinterpret_cast<const char*>(&patchCRC), 4);
}
};