Task 191: .ESF File Format
Task 191: .ESF File Format
File Format Specifications for .ESF
The .ESF file format refers to the Empire Total War Object Serialization Format, a binary container format used in Total War games (starting from Empire: Total War) for storing game data such as savegames, campaign settings, maps, and other serialized objects. It is a tree-like structure of nodes, similar to XML, with a header, body (nodes), and footer. The format has variants identified by magic numbers (0xABCD, 0xABCE, 0xABCF, 0xABCA). All data is little-endian. The specifications are based on detailed reverse-engineering notes from sources like taw's blog and Total War Center forums.
Key structure:
- Header: Magic (4 bytes), optional zeros (4 bytes), optional timestamp (4 bytes), footer offset (4 bytes).
- Body: Root node containing nested nodes (records, arrays, primitive values).
- Footer: String tables (node names, types).
- Nodes have types (byte identifier), optional name index (uint16), optional version (uint8), and content based on type.
- Strings: Prefixed by uint16 length; ASCII (char8) or Unicode (char16).
- Variable integers: uintvar for compact unsigned ints.
List of all the properties of this file format intrinsic to its file system:
- Magic number (uint32): Identifies format variant (e.g., 0xABCE).
- Zeros field (uint32, optional): Always 0x00000000 in some variants.
- Timestamp (uint32, optional): Unix-like timestamp in some variants.
- Footer offset (uint32): Pointer to start of footer.
- Node type (uint8): Defines data type (e.g., 0x01 = bool8, 0x04 = int32, 0x0A = float32, 0x0C = XY coords, 0x80 = single record, 0x81 = multi-record, 0x40-0x4D = arrays).
- Name index (uint16, optional): Index into footer string table for node name.
- Version (uint8, optional): Node version for compatibility.
- Content size/length (varies): For arrays/records (e.g., uint32 count, offsets).
- Primitive values: bool8 (0x01), int8 (0x02), int16 (0x03), int32 (0x04), int64 (0x05), uint8 (0x06), uint16 (0x07), uint32 (0x08), uint64 (0x09), float32 (0x0A), float64 (0x0B), XY (float32 x2), XYZ (float32 x3), angle (uint16).
- String types: ca_ascii (ASCII string), ca_unicode (UTF-16 string).
- Exotic encodings: int24be/uint24be (big-endian 24-bit), uintvar (variable-length unsigned int).
- Record nodes: Contain child nodes; single (0x80) or multiple instances (0x81).
- Array nodes: Typed arrays (e.g., 0x40 = bool[], 0x44 = int32[], 0x4A = float32[]).
- Footer: Offset lists for string tables (node names, types); each table has uint16 count, then repeated {uint16 length, string data}.
Two direct download links for files of format .ESF:
- http://www.2shared.com/file/5031762/3bd7dd1a/unlock_factions.html (ZIP containing startpos.esf from Empire: Total War mod)
- https://www.moddb.com/mods/colonialism-1600ad-work-in-progress/downloads/colonialism-1600ad-startpos-updated-05052014 (Download page for ZIP containing startpos.esf from Colonialism 1600AD mod; direct mirror link available on page)
Ghost blog embedded HTML JavaScript for drag and drop .ESF file dump:
Drag and drop .ESF file here
- Python class for .ESF open, decode, read, write, print properties:
import struct
import os
class ESFHandler:
def __init__(self, filepath):
self.filepath = filepath
self.data = None
self.header = None
self.nodes = [] # Simplified storage
def read(self):
with open(self.filepath, 'rb') as f:
self.data = f.read()
self._decode()
def _decode(self):
offset = 0
magic = struct.unpack_from('<I', self.data, offset)[0]
offset += 4
zeros = 0
timestamp = 0
if hex(magic) != '0xabcd':
zeros = struct.unpack_from('<I', self.data, offset)[0]
offset += 4
timestamp = struct.unpack_from('<I', self.data, offset)[0]
offset += 4
footer_offset = struct.unpack_from('<I', self.data, offset)[0]
offset += 4
self.header = {'magic': hex(magic), 'zeros': zeros, 'timestamp': timestamp, 'footer_offset': footer_offset}
# Simplified node parsing (example for a few nodes)
while offset < footer_offset:
node_type = struct.unpack_from('<B', self.data, offset)[0]
offset += 1
name_index = struct.unpack_from('<H', self.data, offset)[0]
offset += 2
version = struct.unpack_from('<B', self.data, offset)[0]
offset += 1
data = None
if node_type == 0x01: # bool8
data = struct.unpack_from('<B', self.data, offset)[0] != 0
offset += 1
elif node_type == 0x04: # int32
data = struct.unpack_from('<i', self.data, offset)[0]
offset += 4
elif node_type == 0x0A: # float32
data = struct.unpack_from('<f', self.data, offset)[0]
offset += 4
elif node_type == 0x0F: # ascii string
len_ = struct.unpack_from('<H', self.data, offset)[0]
offset += 2
data = self.data[offset:offset + len_].decode('ascii')
offset += len_
# Add more as needed; full parser would recurse for records/arrays
self.nodes.append({'type': hex(node_type), 'name_index': name_index, 'version': version, 'data': data})
def print_properties(self):
print("Header:")
for k, v in self.header.items():
print(f"{k}: {v}")
print("\nNodes (sample):")
for node in self.nodes[:10]: # Limit for console
print(node)
def write(self, new_filepath=None):
if not new_filepath:
new_filepath = self.filepath + '.new'
with open(new_filepath, 'wb') as f:
# Simplified write: rewrite original data (full impl would rebuild from structures)
f.write(self.data)
print(f"Written to {new_filepath}")
# Example usage:
# handler = ESFHandler('path/to/file.esf')
# handler.read()
# handler.print_properties()
# handler.write()
- Java class for .ESF open, decode, read, write, print properties:
import java.io.*;
import java.nio.*;
import java.nio.channels.FileChannel;
public class ESFHandler {
private String filepath;
private ByteBuffer buffer;
private byte[] data;
private String magic;
private int zeros;
private int timestamp;
private int footerOffset;
public ESFHandler(String filepath) {
this.filepath = filepath;
}
public void read() throws IOException {
File file = new File(filepath);
data = new byte[(int) file.length()];
try (FileInputStream fis = new FileInputStream(file)) {
fis.read(data);
}
buffer = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN);
decode();
}
private void decode() {
int magicInt = buffer.getInt();
magic = Integer.toHexString(magicInt).toUpperCase();
if (!"ABCD".equals(magic)) {
zeros = buffer.getInt();
timestamp = buffer.getInt();
}
footerOffset = buffer.getInt();
// Simplified node parsing (example)
System.out.println("Sample nodes:");
for (int i = 0; i < 5 && buffer.position() < footerOffset; i++) {
byte type = buffer.get();
short nameIndex = buffer.getShort();
byte version = buffer.get();
Object dataVal = null;
if (type == 0x01) {
dataVal = buffer.get() != 0;
} else if (type == 0x04) {
dataVal = buffer.getInt();
} else if (type == 0x0A) {
dataVal = buffer.getFloat();
} else if (type == 0x0F) {
short len = buffer.getShort();
byte[] strBytes = new byte[len];
buffer.get(strBytes);
dataVal = new String(strBytes, "ASCII");
}
System.out.println("Type: 0x" + Integer.toHexString(type) + ", NameIndex: " + nameIndex + ", Version: " + version + ", Data: " + dataVal);
}
}
public void printProperties() {
System.out.println("Magic: 0x" + magic);
System.out.println("Zeros: " + zeros);
System.out.println("Timestamp: " + timestamp);
System.out.println("Footer Offset: " + footerOffset);
}
public void write(String newFilepath) throws IOException {
try (FileOutputStream fos = new FileOutputStream(newFilepath)) {
fos.write(data); // Simplified: write original
}
}
// Example usage:
// ESFHandler handler = new ESFHandler("path/to/file.esf");
// handler.read();
// handler.printProperties();
// handler.write("new.esf");
}
- JavaScript class for .ESF open, decode, read, write, print properties (Node.js example):
const fs = require('fs');
class ESFHandler {
constructor(filepath) {
this.filepath = filepath;
this.buffer = null;
this.offset = 0;
this.header = {};
}
read() {
this.buffer = fs.readFileSync(this.filepath);
this.decode();
}
readUint32() {
const val = this.buffer.readUInt32LE(this.offset);
this.offset += 4;
return val;
}
readUint16() {
const val = this.buffer.readUInt16LE(this.offset);
this.offset += 2;
return val;
}
readUint8() {
const val = this.buffer.readUInt8(this.offset);
this.offset += 1;
return val;
}
readFloat32() {
const val = this.buffer.readFloatLE(this.offset);
this.offset += 4;
return val;
}
readString(len, encoding = 'ascii') {
const str = this.buffer.slice(this.offset, this.offset + len).toString(encoding);
this.offset += len;
return str;
}
decode() {
const magic = this.readUint32().toString(16).toUpperCase();
let zeros = 0, timestamp = 0;
if (magic !== 'ABCD') {
zeros = this.readUint32();
timestamp = this.readUint32();
}
const footerOffset = this.readUint32();
this.header = { magic: `0x${magic}`, zeros, timestamp, footerOffset };
// Simplified node parsing
const nodes = [];
while (this.offset < footerOffset) {
const type = this.readUint8();
const nameIndex = this.readUint16();
const version = this.readUint8();
let data = null;
if (type === 0x01) {
data = this.readUint8() !== 0;
} else if (type === 0x04) {
data = this.readUint32();
} else if (type === 0x0A) {
data = this.readFloat32();
} else if (type === 0x0F) {
const len = this.readUint16();
data = this.readString(len);
}
nodes.push({ type: `0x${type.toString(16)}`, nameIndex, version, data });
if (nodes.length >= 10) break; // Limit
}
console.log('Nodes (sample):', nodes);
}
printProperties() {
console.log('Header:', this.header);
}
write(newFilepath) {
fs.writeFileSync(newFilepath || this.filepath + '.new', this.buffer);
}
}
// Example usage:
// const handler = new ESFHandler('path/to/file.esf');
// handler.read();
// handler.printProperties();
// handler.write();
- C class (using C++ for class support) for .ESF open, decode, read, write, print properties:
#include <iostream>
#include <fstream>
#include <vector>
#include <string>
#include <cstdint>
#include <iomanip>
class ESFHandler {
private:
std::string filepath;
std::vector<uint8_t> data;
uint32_t magic;
uint32_t zeros;
uint32_t timestamp;
uint32_t footerOffset;
public:
ESFHandler(const std::string& fp) : filepath(fp), zeros(0), timestamp(0) {}
void read() {
std::ifstream file(filepath, std::ios::binary | std::ios::ate);
std::streamsize size = file.tellg();
file.seekg(0, std::ios::beg);
data.resize(size);
file.read(reinterpret_cast<char*>(data.data()), size);
decode();
}
private:
void decode() {
size_t offset = 0;
magic = *reinterpret_cast<uint32_t*>(&data[offset]);
offset += 4;
if (magic != 0xABCD) {
zeros = *reinterpret_cast<uint32_t*>(&data[offset]);
offset += 4;
timestamp = *reinterpret_cast<uint32_t*>(&data[offset]);
offset += 4;
}
footerOffset = *reinterpret_cast<uint32_t*>(&data[offset]);
offset += 4;
// Simplified node parsing
std::cout << "Sample nodes:" << std::endl;
for (int i = 0; i < 5 && offset < footerOffset; ++i) {
uint8_t type = data[offset++];
uint16_t nameIndex = *reinterpret_cast<uint16_t*>(&data[offset]);
offset += 2;
uint8_t version = data[offset++];
std::string dataStr = "";
if (type == 0x01) {
dataStr = (data[offset++] != 0) ? "true" : "false";
} else if (type == 0x04) {
int32_t val = *reinterpret_cast<int32_t*>(&data[offset]);
offset += 4;
dataStr = std::to_string(val);
} else if (type == 0x0A) {
float val = *reinterpret_cast<float*>(&data[offset]);
offset += 4;
dataStr = std::to_string(val);
} else if (type == 0x0F) {
uint16_t len = *reinterpret_cast<uint16_t*>(&data[offset]);
offset += 2;
dataStr = std::string(&data[offset], &data[offset] + len);
offset += len;
}
std::cout << "Type: 0x" << std::hex << static_cast<int>(type) << ", NameIndex: " << nameIndex << ", Version: " << static_cast<int>(version) << ", Data: " << dataStr << std::endl;
}
}
public:
void printProperties() {
std::cout << "Magic: 0x" << std::hex << magic << std::endl;
std::cout << "Zeros: " << std::dec << zeros << std::endl;
std::cout << "Timestamp: " << timestamp << std::endl;
std::cout << "Footer Offset: " << footerOffset << std::endl;
}
void write(const std::string& newFilepath) {
std::ofstream file(newFilepath, std::ios::binary);
file.write(reinterpret_cast<const char*>(data.data()), data.size());
}
};
// Example usage:
// int main() {
// ESFHandler handler("path/to/file.esf");
// handler.read();
// handler.printProperties();
// handler.write("new.esf");
// return 0;
// }