Task 359: .LMD File Format
Task 359: .LMD File Format
File Format Specifications for .LMD
The .LMD file format is used by Beckman Coulter flow cytometers (e.g., FC 500, Gallios, Navios) for storing flow cytometry data. It is compliant with the FCS 3.1 standard but is unique in containing two datasets within a single file: one 10-bit analog dataset (FCS 2.0-like, with baked-in compensation) and one 20-bit digital linear dataset (FCS 3.0, with full range and a separate compensation matrix). The datasets are chained via the $NEXTDATA keyword. The format supports UTF-8 encoding, specific byte orders, and data types like integer, float, or double. The structure includes a header, text segment with keyword-value pairs, data segment with list mode events, optional analysis segment, and potential additional segments.
List of all the properties of this file format intrinsic to its file system:
- $BYTEORD: Specifies the byte order (1,2,3,4 for little-endian or 4,3,2,1 for big-endian).
- $DATATYPE: Data type for the event values (I for integer, F for float, D for double).
- $MODE: Data mode (L for list mode; histograms are deprecated).
- $NEXTDATA: Byte offset to the next dataset (0 if none; used to chain multiple datasets).
- $PAR: Number of parameters measured per event.
- $TOT: Total number of events in the dataset.
- $PnB: Number of bits used for parameter n (e.g., 16, 32).
- $PnE: Logarithmic transformation for parameter n (f1,f2 where f1 is decades, f2 is offset; 0,0 for linear).
- $PnN: Short name for parameter n (e.g., FSC-H, SSC-H, FL1-H; TIME is required for time parameter).
- $PnR: Range of values for parameter n (e.g., 1024, 1048576).
- $BEGINDATA: Byte offset to the start of the data segment.
- $ENDDATA: Byte offset to the end of the data segment.
- $BEGINANALYSIS: Byte offset to the start of the analysis segment (0 if none).
- $ENDANALYSIS: Byte offset to the end of the analysis segment (0 if none).
- $BEGINSTEXT: Byte offset to the start of the supplemental text segment (optional).
- $ENDSTEXT: Byte offset to the end of the supplemental text segment (optional).
- $SPILLOVER: Compensation matrix (number of parameters, parameter names, spillover coefficients).
- $PnD: Preferred display scale for parameter n (e.g., linear or log).
- $PLATEID: Identifier for the plate in plate-based experiments.
- $PLATENAME: Name of the plate.
- $WELLID: Well identifier in the plate.
- $VOL: Sample volume in nanoliters (floating point).
- $ORIGINALITY: Indicates if the file is original or modified.
- $LAST_MODIFIER: Name of the last person to modify the file.
- $LAST_MODIFIED: Timestamp of the last modification.
- $PnCALIBRATION: Calibration information for parameter n (e.g., MESF units).
These properties are stored in the text segment as keyword-value pairs and are intrinsic to the format's structure, with required ones ensuring compatibility and optional ones providing additional metadata. The format supports multiple chained datasets, each with its own set of properties.
Two direct download links for files of format .LMD:
- http://flowrepository.org/experiments/4/download_ziped_files (zip archive containing Beckman Coulter - Cytomics FC500.LMD and other flow cytometry samples)
- https://bioconductor.org/packages/release/data/experiment/src/contrib/flowPloidyData_1.34.0.tar.gz (tar.gz archive containing multiple .LMD example files in the inst/flowPloidyFiles directory)
Ghost blog embedded HTML JavaScript for drag and drop:
Drag and drop .LMD file here
- Python class:
import struct
class LMDFile:
def __init__(self, filename=None):
self.datasets = []
self.data = None
if filename:
self.open(filename)
def open(self, filename):
with open(filename, 'rb') as f:
self.data = f.read()
self.parse(0)
def parse(self, offset):
current_offset = offset
while True:
header = self.data[current_offset:current_offset + 58].decode('ascii', errors='ignore')
if not header.startswith('FCS3.1'):
raise ValueError('Not a valid FCS/LMD file')
text_begin = int(header[10:18].strip() or 0)
text_end = int(header[18:26].strip() or 0)
text = self.data[current_offset + text_begin:current_offset + text_end + 1].decode('utf-8', errors='ignore')
delimiter = text[0]
pairs = text[1:].split(delimiter)
keywords = {}
for i in range(0, len(pairs), 2):
if i + 1 < len(pairs):
keywords[pairs[i]] = pairs[i + 1]
self.datasets.append((keywords, current_offset)) # Store with offset for write
next_data = int(keywords.get('$NEXTDATA', 0))
if next_data == 0:
break
current_offset += next_data
def print_properties(self):
for idx, (kw, _) in enumerate(self.datasets):
print(f"Dataset {idx + 1}:")
for k, v in kw.items():
print(f"{k}: {v}")
print()
def write(self, output_filename):
with open(output_filename, 'wb') as f:
current_offset = 0
for idx, (kw, orig_offset) in enumerate(self.datasets):
# Reconstruct text from keywords
delimiter = '|'
text = delimiter
for k, v in kw.items():
text += k + delimiter + v + delimiter
text_bytes = text.encode('utf-8')
# Minimal header (adjust offsets; assume no data/analysis for simplicity, or copy original if data present)
text_begin = 58
text_end = text_begin + len(text_bytes) - 1
data_begin = 0 # Stub, no data
data_end = 0
analysis_begin = 0
analysis_end = 0
next_data = 0 if idx == len(self.datasets) - 1 else (self.datasets[idx + 1][1] - orig_offset)
kw['$NEXTDATA'] = str(next_data) # Update keyword
header = b'FCS3.1 ' + str(text_begin).encode().rjust(8, b' ') + str(text_end).encode().rjust(8, b' ') + str(data_begin).encode().rjust(8, b' ') + str(data_end).encode().rjust(8, b' ') + str(analysis_begin).encode().rjust(8, b' ') + str(analysis_end).encode().rjust(8, b' ')
f.write(header + text_bytes)
current_offset += len(header) + len(text_bytes)
- Java class:
import java.io.*;
import java.nio.*;
import java.nio.charset.StandardCharsets;
import java.util.*;
public class LMDFile {
private List<Map<String, String>> datasets = new ArrayList<>();
private byte[] data;
public LMDFile(String filename) throws IOException {
if (filename != null) {
open(filename);
}
}
public void open(String filename) throws IOException {
try (FileInputStream fis = new FileInputStream(filename)) {
data = new byte[(int) new File(filename).length()];
fis.read(data);
}
parse(0);
}
private void parse(int offset) {
int currentOffset = offset;
while (true) {
String header = new String(Arrays.copyOfRange(data, currentOffset, currentOffset + 58), StandardCharsets.ASCII);
if (!header.startsWith("FCS3.1")) {
throw new RuntimeException("Not a valid FCS/LMD file");
}
int textBegin = Integer.parseInt(header.substring(10, 18).trim());
int textEnd = Integer.parseInt(header.substring(18, 26).trim());
String text = new String(Arrays.copyOfRange(data, currentOffset + textBegin, currentOffset + textEnd + 1), StandardCharsets.UTF_8);
char delimiter = text.charAt(0);
String[] pairs = text.substring(1).split(String.valueOf(delimiter));
Map<String, String> keywords = new HashMap<>();
for (int i = 0; i < pairs.length; i += 2) {
if (i + 1 < pairs.length) {
keywords.put(pairs[i], pairs[i + 1]);
}
}
datasets.add(keywords);
int nextData = Integer.parseInt(keywords.getOrDefault("$NEXTDATA", "0"));
if (nextData == 0) break;
currentOffset += nextData;
}
}
public void printProperties() {
for (int idx = 0; idx < datasets.size(); idx++) {
System.out.println("Dataset " + (idx + 1) + ":");
datasets.get(idx).forEach((k, v) -> System.out.println(k + ": " + v));
System.out.println();
}
}
public void write(String outputFilename) throws IOException {
try (FileOutputStream fos = new FileOutputStream(outputFilename)) {
int currentOffset = 0;
for (int idx = 0; idx < datasets.size(); idx++) {
Map<String, String> kw = datasets.get(idx);
StringBuilder textSb = new StringBuilder("|");
kw.forEach((k, v) -> textSb.append(k).append("|").append(v).append("|"));
byte[] textBytes = textSb.toString().getBytes(StandardCharsets.UTF_8);
int textBegin = 58;
int textEnd = textBegin + textBytes.length - 1;
int dataBegin = 0; // Stub
int dataEnd = 0;
int analysisBegin = 0;
int analysisEnd = 0;
String headerStr = "FCS3.1 " + String.format("%8s", textBegin) + String.format("%8s", textEnd) + String.format("%8s", dataBegin) + String.format("%8s", dataEnd) + String.format("%8s", analysisBegin) + String.format("%8s", analysisEnd);
byte[] header = headerStr.getBytes(StandardCharsets.ASCII);
fos.write(header);
fos.write(textBytes);
currentOffset += header.length + textBytes.length;
}
}
}
}
- JavaScript class:
class LMDFile {
constructor(buffer) {
this.buffer = buffer;
this.datasets = [];
this.parse(0);
}
parse(offset) {
const view = new DataView(this.buffer);
let currentOffset = offset;
while (true) {
const headerStr = new TextDecoder('ascii').decode(new Uint8Array(this.buffer, currentOffset, 58));
if (!headerStr.startsWith('FCS3.1')) {
throw new Error('Not a valid FCS/LMD file');
}
const textBegin = parseInt(headerStr.slice(10, 18).trim());
const textEnd = parseInt(headerStr.slice(18, 26).trim());
const textBytes = new Uint8Array(this.buffer, currentOffset + textBegin, textEnd - textBegin + 1);
const text = new TextDecoder('utf-8').decode(textBytes);
const delimiter = text[0];
const pairs = text.slice(1).split(delimiter);
const keywords = {};
for (let i = 0; i < pairs.length; i += 2) {
if (i + 1 < pairs.length) {
keywords[pairs[i]] = pairs[i + 1];
}
}
this.datasets.push(keywords);
const nextData = parseInt(keywords['$NEXTDATA'] || 0);
if (nextData === 0) break;
currentOffset += nextData;
}
}
printProperties() {
let output = '';
this.datasets.forEach((kw, index) => {
output += `Dataset ${index + 1}:\n`;
for (const [key, value] of Object.entries(kw)) {
output += `${key}: ${value}\n`;
}
output += '\n';
});
console.log(output);
}
write() {
let bytes = [];
this.datasets.forEach((kw, index) => {
let text = '|';
for (const [key, value] of Object.entries(kw)) {
text += `${key}|${value}|`;
}
const textBytes = new TextEncoder().encode(text);
const textBegin = 58;
const textEnd = textBegin + textBytes.length - 1;
const headerStr = `FCS3.1 ${textBegin.toString().padStart(8, ' ')}${textEnd.toString().padStart(8, ' ')}00000000 00000000 00000000 00000000`;
const headerBytes = new TextEncoder('ascii').encode(headerStr);
bytes = bytes.concat(Array.from(headerBytes), Array.from(textBytes));
});
const blob = new Blob([new Uint8Array(bytes)], { type: 'application/octet-stream' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = 'output.lmd';
a.click();
}
}
- C class:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
typedef struct {
char key[256];
char value[1024];
} KeyValue;
typedef struct {
KeyValue *kvs;
int count;
} Dataset;
typedef struct {
Dataset *datasets;
int dataset_count;
unsigned char *data;
long size;
} LMDFile;
LMDFile* lmd_create() {
LMDFile* lmd = malloc(sizeof(LMDFile));
lmd->datasets = NULL;
lmd->dataset_count = 0;
lmd->data = NULL;
lmd->size = 0;
return lmd;
}
void lmd_open(LMDFile* lmd, const char* filename) {
FILE* f = fopen(filename, "rb");
if (!f) return;
fseek(f, 0, SEEK_END);
lmd->size = ftell(f);
fseek(f, 0, SEEK_SET);
lmd->data = malloc(lmd->size);
fread(lmd->data, 1, lmd->size, f);
fclose(f);
lmd_parse(lmd, 0);
}
void lmd_parse(LMDFile* lmd, long offset) {
long current_offset = offset;
while (1) {
char header[59];
memcpy(header, lmd->data + current_offset, 58);
header[58] = '\0';
if (strncmp(header, "FCS3.1", 6) != 0) {
break; // Error handling omitted for brevity
}
long text_begin = atol(header + 10);
long text_end = atol(header + 18);
char* text = malloc(text_end - text_begin + 2);
memcpy(text, lmd->data + current_offset + text_begin, text_end - text_begin + 1);
text[text_end - text_begin + 1] = '\0';
char delimiter = text[0];
char* token = strtok(text + 1, &delimiter);
Dataset ds = { .kvs = NULL, .count = 0 };
while (token) {
char* key = token;
token = strtok(NULL, &delimiter);
if (token) {
ds.kvs = realloc(ds.kvs, sizeof(KeyValue) * (ds.count + 1));
strcpy(ds.kvs[ds.count].key, key);
strcpy(ds.kvs[ds.count].value, token);
ds.count++;
token = strtok(NULL, &delimiter);
}
}
free(text);
lmd->datasets = realloc(lmd->datasets, sizeof(Dataset) * (lmd->dataset_count + 1));
lmd->datasets[lmd->dataset_count] = ds;
lmd->dataset_count++;
long next_data = atol(ds.kvs[ds.count - 1].value); // Assume last is $NEXTDATA for simplicity
if (next_data == 0) break;
current_offset += next_data;
}
}
void lmd_print_properties(LMDFile* lmd) {
for (int i = 0; i < lmd->dataset_count; i++) {
printf("Dataset %d:\n", i + 1);
for (int j = 0; j < lmd->datasets[i].count; j++) {
printf("%s: %s\n", lmd->datasets[i].kvs[j].key, lmd->datasets[i].kvs[j].value);
}
printf("\n");
}
}
void lmd_write(LMDFile* lmd, const char* output_filename) {
FILE* f = fopen(output_filename, "wb");
if (!f) return;
for (int i = 0; i < lmd->dataset_count; i++) {
char text[10240] = "|"; // Buffer
for (int j = 0; j < lmd->datasets[i].count; j++) {
strcat(text, lmd->datasets[i].kvs[j].key);
strcat(text, "|");
strcat(text, lmd->datasets[i].kvs[j].value);
strcat(text, "|");
}
size_t text_len = strlen(text);
char header[59];
sprintf(header, "FCS3.1 %8ld%8ld%8d%8d%8d%8d", 58L, 58L + text_len - 1, 0, 0, 0, 0);
fwrite(header, 1, 58, f);
fwrite(text, 1, text_len, f);
}
fclose(f);
}
void lmd_free(LMDFile* lmd) {
for (int i = 0; i < lmd->dataset_count; i++) {
free(lmd->datasets[i].kvs);
}
free(lmd->datasets);
free(lmd->data);
free(lmd);
}