Talk:CXML Containers: Difference between revisions

From PS3 Developer wiki
Jump to navigation Jump to search
(moved to front)
Line 1: Line 1:


=Tools=
==CXML decompiler==
Experimental tool coded by flatz for documenting purposes, three versions of the tool was released publically in a IRC channel the same day (first and second versions had bugs which were identified and fixed)
The tool can extract the files contained inside a CXML container (CXML, QRCF, P3TF, RAFO, etc...), and generates an .xml that represents the original CXML structure
*Usage:
**Note the original files [[lines.qrc]] and [[coldboot.raf]] are compressed with zlib (a compression layer that affects the whole file and makes it imposible to read the cxml structure). The command line examples below considers the files was decompressed in a previous step (using a [[Qt_Resource_Container_%28QRC%29#ZLIB_archivers|zlib archiver]]), in this previous step the file-extension was renamed to "qrcf" and "rafo" (which are the decompressed versions where the cxml structure is fully readable and the tool can process it)
{{Keyboard|content=
'''C:\Portables\cxml decompiler v3 alpha>decompiler.exe lines.qrcf lines.xml'''
}}
{{Keyboard|content=
'''C:\Portables\cxml decompiler v3 alpha>decompiler.exe coldboot.rafo coldboot.xml'''
}}
*CXML decompiler v3 alpha, download links: http://multiupload.biz/dnsipf0dkssz/cxml_decompiler_v3_alpha_MultiUpload.biz.7z.html
'''Changelog'''
'''---------'''
v1 alpha - Basic .cxml support
v2 alpha - Fixed a bug related with RAFO header (coldboot.raf support added)
v3 alpha - Fixed offset/length displacements in the function that locates float values (xml tag attributes related with x,y,z axis in 3D space now looks correct)
*CXML decompiler v3 alpha, source code:
<div style="height:600px; overflow:auto">
{{Boxcode|content=<syntaxhighlight lang="python">
#!python2
import sys, os, struct
from io import BytesIO
from pprint import pprint
def read_cstring(f):
bytes = []
while True:
byte = f.read(1)
if byte == b'\x00':
break
elif byte == '':
raise EOFError()
else:
bytes.append(byte)
return b''.join(bytes)
def check_file_magic(f, expected_magic):
old_offset = f.tell()
try:
magic = f.read(len(expected_magic))
except:
return False
finally:
f.seek(old_offset)
return magic == expected_magic
script_file_name = os.path.split(sys.argv[0])[1]
script_file_base = os.path.splitext(script_file_name)[0]
if len(sys.argv) < 2:
print('CXML decompiler (c) flatz')
print('Usage: {0} <cxml file> <xml file>'.format(script_file_name))
sys.exit()
ENDIANNESS = '>'
def write_raw(f, data):
if type(data) == str:
f.write(data)
elif type(data) == unicode:
f.write(data.decode('utf-8'))
else:
f.write(data)
def write_indent(f, depth):
write_raw(f, '\t' * depth)
def write_line(f, data):
write_raw(f, data)
write_raw(f, '\n')
INT_FMT = ENDIANNESS + 'i'
FLOAT_FMT = ENDIANNESS + 'f'
STRING_FMT = ENDIANNESS + 'ii'
INT_ARRAY_FMT = ENDIANNESS + 'ii'
FLOAT_ARRAY_FMT = ENDIANNESS + 'ii'
FILE_FMT = ENDIANNESS + 'ii'
ID_FMT = ENDIANNESS + 'i'
ID_REF_FMT = ENDIANNESS + 'i'
class Attribute(object):
HEADER_FMT = ENDIANNESS + 'ii'
HEADER_SIZE = struct.calcsize(HEADER_FMT)
SIZE = HEADER_SIZE + max(struct.calcsize(INT_FMT), struct.calcsize(FLOAT_FMT), struct.calcsize(STRING_FMT), struct.calcsize(INT_ARRAY_FMT), struct.calcsize(FLOAT_ARRAY_FMT), struct.calcsize(FILE_FMT), struct.calcsize(ID_FMT), struct.calcsize(ID_REF_FMT))
TYPE_NONE = 0
TYPE_INT = 1
TYPE_FLOAT = 2
TYPE_STRING = 3
TYPE_INT_ARRAY = 4
TYPE_FLOAT_ARRAY = 5
TYPE_FILE = 6
TYPE_ID = 7
TYPE_ID_REF = 8
def __init__(self, element):
self.element = element
self.start = None
self.name = None
self.type = None
self.offset = None
self.length = None
self.value = None
def load(self, f):
self.start = f.tell()
data = f.read(self.HEADER_SIZE)
self.name, self.type = struct.unpack(self.HEADER_FMT, data)
data = f.read(self.SIZE - self.HEADER_SIZE)
if self.type == self.TYPE_NONE:
pass
elif self.type == self.TYPE_INT:
self.value, = struct.unpack(INT_FMT, data[:struct.calcsize(INT_FMT)])
elif self.type == self.TYPE_FLOAT:
self.value, = struct.unpack(FLOAT_FMT, data[:struct.calcsize(FLOAT_FMT)])
elif self.type == self.TYPE_STRING:
self.offset, self.length = struct.unpack(STRING_FMT, data[:struct.calcsize(STRING_FMT)])
elif self.type == self.TYPE_INT_ARRAY:
self.offset, self.length = struct.unpack(INT_ARRAY_FMT, data[:struct.calcsize(INT_ARRAY_FMT)])
elif self.type == self.TYPE_FLOAT_ARRAY:
self.offset, self.length = struct.unpack(FLOAT_ARRAY_FMT, data[:struct.calcsize(FLOAT_ARRAY_FMT)])
elif self.type == self.TYPE_FILE:
self.offset, self.length = struct.unpack(FILE_FMT, data[:struct.calcsize(FILE_FMT)])
elif self.type == self.TYPE_ID:
self.offset, = struct.unpack(ID_FMT, data[:struct.calcsize(ID_FMT)])
elif self.type == self.TYPE_ID_REF:
self.offset, = struct.unpack(ID_REF_FMT, data[:struct.calcsize(ID_REF_FMT)])
return True
def get_name(self):
return self.element.document.get_string(self.name)
def get_int(self):
if self.type != self.TYPE_INT:
return None
return self.value
def get_float(self):
if self.type != self.TYPE_FLOAT:
return None
return self.value
def get_string(self):
if self.type != self.TYPE_STRING:
return None
value = self.element.document.get_string(self.offset)
if len(value) != self.length:
return None
return value
def get_int_array(self):
if self.type != self.TYPE_INT_ARRAY:
return None
value = self.element.document.get_int_array(self.offset, self.length)
if len(value) != self.length:
return None
return value
def get_float_array(self):
if self.type != self.TYPE_FLOAT_ARRAY:
return None
value = self.element.document.get_float_array(self.offset, self.length)
if len(value) != self.length:
return None
return value
def get_file(self):
if self.type != self.TYPE_FILE:
return None
value = self.element.document.get_file(self.offset, self.length)
return value
def get_id(self):
if self.type != self.TYPE_ID:
return None
id = self.element.document.get_id_string(self.offset)
return id
def get_id_ref(self):
if self.type != self.TYPE_ID_REF:
return None
id = self.element.document.get_id_string(self.offset)
element = Element(self.element.document)
return [id, element]
def dump(self, f, depth):
pass
#print('  ' * depth + 'Attribute:' + 'name:{0} type:{1}'.format(self.name, self.type), end='\n', file=f)
class Element(object):
HEADER_FMT = ENDIANNESS + 'iiiiiii'
SIZE = struct.calcsize(HEADER_FMT)
TAG_NAME = 0
ATTR_NUM = 1
PARENT = 2
PREV = 3
NEXT = 4
FIRST_CHILD = 5
LAST_CHILD = 6
def __init__(self, document):
self.document = document
self.start = None
self.name = None
self.num_attributes = None
self.parent = None
self.prev = None
self.next = None
self.first_child = None
self.last_child = None
def load(self, f):
self.start = f.tell()
self.name, self.num_attributes, self.parent, self.prev, self.next, self.first_child, self.last_child = struct.unpack(self.HEADER_FMT, f.read(self.SIZE))
return True
def get_name(self):
return self.document.get_string(self.name)
def get_attribute(self, index):
if index < 0 or index >= self.num_attributes:
return None
offset = self.start + Element.SIZE + index * Attribute.SIZE
if not is_valid_attribute(self.document, offset):
return None
attribute = Attribute(self)
f = BytesIO(self.document.tree_bin)
f.seek(offset)
attribute.load(f)
return attribute
def get_parent(self):
if not is_valid_element(self.document, self.parent):
return None
element = Element(self.document)
f = BytesIO(self.document.tree_bin)
f.seek(parent)
element.load(f)
return element
def get_first_child(self):
if not is_valid_element(self.document, self.first_child):
return None
element = Element(self.document)
f = BytesIO(self.document.tree_bin)
f.seek(self.first_child)
element.load(f)
return element
def get_last_child(self):
if not is_valid_element(self.document, self.last_child):
return None
element = Element(self.document)
f = BytesIO(self.document.tree_bin)
f.seek(self.last_child)
element.load(f)
return element
def get_prev_sibling(self):
if not is_valid_element(self.document, self.prev):
return None
element = Element(self.document)
f = BytesIO(self.document.tree_bin)
f.seek(self.prev)
element.load(f)
return element
def get_next_sibling(self):
if not is_valid_element(self.document, self.next):
return None
element = Element(self.document)
f = BytesIO(self.document.tree_bin)
f.seek(self.next)
element.load(f)
return element
def dump(self, f, depth):
write_indent(f, depth)
name = self.get_name()
write_raw(f, '<' + name)
for i in range(self.num_attributes):
attribute = self.get_attribute(i)
if attribute is None:
return False
write_raw(f, ' {0}='.format(attribute.get_name()))
if attribute.type == Attribute.TYPE_NONE:
write_raw(f, '\"null\"')
elif attribute.type == Attribute.TYPE_INT:
write_raw(f, '\"{0}\"'.format(attribute.get_int()))
elif attribute.type == Attribute.TYPE_FLOAT:
write_raw(f, '\"{0:3.6}\"'.format(attribute.get_float()))
elif attribute.type == Attribute.TYPE_STRING:
write_raw(f, '\"{0}\"'.format(attribute.get_string()))
elif attribute.type == Attribute.TYPE_INT_ARRAY:
write_raw(f, '\"')
array = attribute.get_int_array()
array_length = len(array)
for j in range(array_length):
write_raw(f, '{0}'.format(array[j]))
if j + 1 < array_length:
write_raw(f, ',')
write_raw(f, '\"')
elif attribute.type == Attribute.TYPE_FLOAT_ARRAY:
write_raw(f, '\"')
array = attribute.get_float_array()
array_length = len(array)
for j in range(array_length):
write_raw(f, '{0:3.6}'.format(array[j]))
if j + 1 < array_length:
write_raw(f, ',')
write_raw(f, '\"')
elif attribute.type == Attribute.TYPE_FILE:
file_name = '{0}_0x{1:08X}.bin'.format(self.document.file_prefix, attribute.offset)
file_data = attribute.get_file()
with open(file_name, 'wb') as of:
of.write(file_data)
write_raw(f, '\"{0}\"'.format(file_name))
elif attribute.type == Attribute.TYPE_ID:
write_raw(f, '\"{0}\"'.format(attribute.get_id()))
elif attribute.type == Attribute.TYPE_ID_REF:
id_entity = attribute.get_id_ref()
write_raw(f, '\"{0}\"'.format(id_entity[0]))
child_element = self.get_first_child()
if not child_element is None:
write_raw(f, '>\n')
while not child_element is None:
child_element.dump(f, depth + 1)
child_element = child_element.get_next_sibling()
write_indent(f, depth)
write_raw(f, '</' + name + '>\n')
else:
write_raw(f, ' />\n')
def is_valid_element(document, offset):
if offset < 0 or offset + Element.SIZE > document.tree_size:
return False
element = Element(document)
f = BytesIO(document.tree_bin)
f.seek(offset)
element.load(f)
if element.num_attributes < 0 or offset + Element.SIZE + element.num_attributes * Attribute.SIZE > document.tree_size:
return False
return True
def is_valid_attribute(document, offset):
if offset < 0 or offset + Attribute.SIZE > document.tree_size:
return False
return True
class Document(object):
HEADER_FMT = ENDIANNESS + '4siiiiiiiiiiiii8x'
HEADER_SIZE = struct.calcsize(HEADER_FMT)
def __init__(self, file_prefix=''):
self.file_prefix = file_prefix
self.magic = None
self.version = None
self.tree_offset = None
self.tree_size = None
self.id_table_offset = None
self.id_table_size = None
self.string_table_offset = None
self.string_table_size = None
self.int_array_table_offset = None
self.int_array_table_size = None
self.float_array_table_offset = None
self.float_array_table_size = None
self.file_table_offset = None
self.file_table_size = None
self.tree_bin = None
self.id_table_bin = None
self.string_table_bin = None
self.int_array_table_bin = None
self.float_array_table_bin = None
self.file_table_bin = None
self.root = None
def get_document_element(self):
if not is_valid_element(self, 0):
return None
element = Element(self)
f = BytesIO(self.tree_bin)
element.load(f)
return element
def get_id_string(self, offset):
if offset < 0 or offset >= self.id_table_size:
return None
f = BytesIO(self.id_table_bin)
f.seek(offset)
entity_offset, = struct.unpack(INT_FMT, f.read(struct.calcsize(INT_FMT)))
return read_cstring(f)
def get_string(self, offset):
if offset < 0 or offset >= self.string_table_size:
return None
f = BytesIO(self.string_table_bin)
f.seek(offset)
return read_cstring(f)
def get_int_array(self, offset, length):
if offset < 0 or (offset + length) * struct.calcsize(INT_FMT) > self.int_array_table_size:
return None
f = BytesIO(self.int_array_table_bin)
f.seek(offset * struct.calcsize(INT_FMT))
array = []
for i in range(length):
value, = struct.unpack(INT_FMT, f.read(struct.calcsize(INT_FMT)))
array.append(value)
return array
def get_float_array(self, offset, length):
if offset < 0 or (offset + length) * struct.calcsize(FLOAT_FMT) > self.float_array_table_size:
return None
f = BytesIO(self.float_array_table_bin)
f.seek(offset * struct.calcsize(FLOAT_FMT))
array = []
for i in range(length):
value, = struct.unpack(FLOAT_FMT, f.read(struct.calcsize(FLOAT_FMT)))
array.append(value)
return array
def get_file(self, offset, length):
if offset < 0 or offset + length > self.file_table_size:
return None
return self.file_table_bin[offset:offset + length]
def load(self, f):
self.magic, self.version, self.tree_offset, self.tree_size, self.id_table_offset, self.id_table_size, self.string_table_offset, self.string_table_size, self.int_array_table_offset, self.int_array_table_size, self.float_array_table_offset, self.float_array_table_size, self.file_table_offset, self.file_table_size = struct.unpack(self.HEADER_FMT, f.read(self.HEADER_SIZE))
f.seek(self.tree_offset)
self.tree_bin = f.read(self.tree_size)
f.seek(self.id_table_offset)
self.id_table_bin = f.read(self.id_table_size)
f.seek(self.string_table_offset)
self.string_table_bin = f.read(self.string_table_size)
f.seek(self.int_array_table_offset)
self.int_array_table_bin = f.read(self.int_array_table_size)
f.seek(self.float_array_table_offset)
self.float_array_table_bin = f.read(self.float_array_table_size)
f.seek(self.file_table_offset)
self.file_table_bin = f.read(self.file_table_size)
self.root = self.get_document_element()
return True
def check(self, f):
return check_file_magic(f, 'CXML')
def dump(self, f=sys.stdout, depth=0):
if self.root is None:
return
self.root.dump(f, depth)
if len(sys.argv) < 3:
print('error: insufficient options specified')
sys.exit()
cxml_file_path = sys.argv[1]
if not os.path.isfile(cxml_file_path):
print('error: invalid cxml file specified')
sys.exit()
xml_file_path = sys.argv[2]
if os.path.exists(xml_file_path) and not os.path.isfile(xml_file_path):
print('error: invalid xml file specified')
sys.exit()
cxml_file_base = os.path.splitext(cxml_file_path)[0]
document = Document(cxml_file_base)
with open(cxml_file_path, 'rb') as f:
#if not document.check(f):
# print 'error: invalid CXML file format'
# sys.exit()
document.load(f)
with open(xml_file_path, 'wb') as f:
write_raw(f, '<?xml version="1.0" encoding="utf-8"?>\n')
document.dump(f)
</syntaxhighlight>}}
</div>

Revision as of 17:32, 8 November 2014

Retrieved from "http://www.psdevwiki.com/ps3/index.php?title=Talk:CXML_Containers&oldid=31374"