Talk:CXML Containers: Difference between revisions

From PS3 Developer wiki
Jump to navigation Jump to search
m (Text replacement - "vitadevwiki.com" to "psdevwiki.com/vita")
 
(15 intermediate revisions by 2 users not shown)
Line 1: Line 1:


=Tools=
=Other sony consoles=
 
*In PSvita the RCO's uses a cxml structure, with a header size of 0x50 and the magic identifyer '''RCOF'''
==CXML decompiler==
**RCOF header is 0x10 bytes bigger than any cxml header in PS3 (this means there are 2 new tables added for the new cxml variant used in vita rco's). And endianess changed because PSVita processor arquitecture
Experimental tool coded by flatz for documenting purposes, three versions of the tool was released publically in a IRC channel the same day (first and second versions had bugs that was identifyed and fixed)
**Some of the extracted files (after cropped and zlib decompressed) are another variant of a cxml structure with identifyer '''RCSF'''
 
**More info about this cxml variants in [http://www.psdevwiki.com/vita/index.php?title=RCO PSVita wiki]
The tool can extract the files contained inside a CXML container (CXML, QRCF, P3TF, RAFO, etc...), and generates an .xml that represents the original CXML structure
 
*Usage:
**Note the original files [[lines.qrc]] and [[coldboot.raf]] are compressed with zlib (a compression layer that affects the whole file and makes imposible to read the cxml structure), the command line examples below considers the files was decompressed in a previous step (using a [http://www.psdevwiki.com/ps3/Qt_Resource_Container_%28QRC%29#ZLIB_archivers zlib archiver]), in this previous step his file-extensions was renamed to "qrcf" and "rafo" (are the decompressed versions where the cxml structure is fully readable and the tool can process it)
 
has been removed from the zlib compression layer that originally is applyed over the whole file... this is why his file-extensions in the command line examples are renamed to "qrcf" and "rafo"
 
{{Keyboard|content=
'''C:\Portables\cxml decompiler v3 alpha>decompiler.exe lines.qrcf lines.xml'''
}}
{{Keyboard|content=
'''C:\Portables\cxml decompiler v3 alpha>decompiler.exe coldboot.rafo coldboot.xml'''
}}
 
*CXML decompiler v3 alpha, download links: http://multiupload.biz/dnsipf0dkssz/cxml_decompiler_v3_alpha_MultiUpload.biz.7z.html
'''Changelog'''
'''---------'''
v1 alpha - Basic .cxml support
v2 alpha - Fixed a bug related with RAFO header (coldboot.raf support added)
v3 alpha - Fixed offset/length displacements in the function that locates float values (xml tag attributes related with x,y,z axis in 3D space now looks correct)
 
*CXML decompiler v3 alpha, source code:
<div style="height:600px; overflow:auto">
{{Boxcode|content=<syntaxhighlight lang="python">
#!python2
 
import sys, os, struct
 
from io import BytesIO
from pprint import pprint
 
def read_cstring(f):
bytes = []
while True:
byte = f.read(1)
if byte == b'\x00':
break
elif byte == '':
raise EOFError()
else:
bytes.append(byte)
return b''.join(bytes)
 
def check_file_magic(f, expected_magic):
old_offset = f.tell()
try:
magic = f.read(len(expected_magic))
except:
return False
finally:
f.seek(old_offset)
return magic == expected_magic
 
script_file_name = os.path.split(sys.argv[0])[1]
script_file_base = os.path.splitext(script_file_name)[0]
 
if len(sys.argv) < 2:
print('CXML decompiler (c) flatz')
print('Usage: {0} <cxml file> <xml file>'.format(script_file_name))
sys.exit()
 
ENDIANNESS = '>'
 
def write_raw(f, data):
if type(data) == str:
f.write(data)
elif type(data) == unicode:
f.write(data.decode('utf-8'))
else:
f.write(data)
def write_indent(f, depth):
write_raw(f, '\t' * depth)
def write_line(f, data):
write_raw(f, data)
write_raw(f, '\n')
 
INT_FMT = ENDIANNESS + 'i'
FLOAT_FMT = ENDIANNESS + 'f'
STRING_FMT = ENDIANNESS + 'ii'
INT_ARRAY_FMT = ENDIANNESS + 'ii'
FLOAT_ARRAY_FMT = ENDIANNESS + 'ii'
FILE_FMT = ENDIANNESS + 'ii'
ID_FMT = ENDIANNESS + 'i'
ID_REF_FMT = ENDIANNESS + 'i'
 
class Attribute(object):
HEADER_FMT = ENDIANNESS + 'ii'
HEADER_SIZE = struct.calcsize(HEADER_FMT)
SIZE = HEADER_SIZE + max(struct.calcsize(INT_FMT), struct.calcsize(FLOAT_FMT), struct.calcsize(STRING_FMT), struct.calcsize(INT_ARRAY_FMT), struct.calcsize(FLOAT_ARRAY_FMT), struct.calcsize(FILE_FMT), struct.calcsize(ID_FMT), struct.calcsize(ID_REF_FMT))
 
TYPE_NONE = 0
TYPE_INT = 1
TYPE_FLOAT = 2
TYPE_STRING = 3
TYPE_INT_ARRAY = 4
TYPE_FLOAT_ARRAY = 5
TYPE_FILE = 6
TYPE_ID = 7
TYPE_ID_REF = 8
 
def __init__(self, element):
self.element = element
self.start = None
self.name = None
self.type = None
self.offset = None
self.length = None
self.value = None
 
def load(self, f):
self.start = f.tell()
data = f.read(self.HEADER_SIZE)
self.name, self.type = struct.unpack(self.HEADER_FMT, data)
data = f.read(self.SIZE - self.HEADER_SIZE)
if self.type == self.TYPE_NONE:
pass
elif self.type == self.TYPE_INT:
self.value, = struct.unpack(INT_FMT, data[:struct.calcsize(INT_FMT)])
elif self.type == self.TYPE_FLOAT:
self.value, = struct.unpack(FLOAT_FMT, data[:struct.calcsize(FLOAT_FMT)])
elif self.type == self.TYPE_STRING:
self.offset, self.length = struct.unpack(STRING_FMT, data[:struct.calcsize(STRING_FMT)])
elif self.type == self.TYPE_INT_ARRAY:
self.offset, self.length = struct.unpack(INT_ARRAY_FMT, data[:struct.calcsize(INT_ARRAY_FMT)])
elif self.type == self.TYPE_FLOAT_ARRAY:
self.offset, self.length = struct.unpack(FLOAT_ARRAY_FMT, data[:struct.calcsize(FLOAT_ARRAY_FMT)])
elif self.type == self.TYPE_FILE:
self.offset, self.length = struct.unpack(FILE_FMT, data[:struct.calcsize(FILE_FMT)])
elif self.type == self.TYPE_ID:
self.offset, = struct.unpack(ID_FMT, data[:struct.calcsize(ID_FMT)])
elif self.type == self.TYPE_ID_REF:
self.offset, = struct.unpack(ID_REF_FMT, data[:struct.calcsize(ID_REF_FMT)])
return True
 
def get_name(self):
return self.element.document.get_string(self.name)
 
def get_int(self):
if self.type != self.TYPE_INT:
return None
return self.value
 
def get_float(self):
if self.type != self.TYPE_FLOAT:
return None
return self.value
 
def get_string(self):
if self.type != self.TYPE_STRING:
return None
value = self.element.document.get_string(self.offset)
if len(value) != self.length:
return None
return value
 
def get_int_array(self):
if self.type != self.TYPE_INT_ARRAY:
return None
value = self.element.document.get_int_array(self.offset, self.length)
if len(value) != self.length:
return None
return value
 
def get_float_array(self):
if self.type != self.TYPE_FLOAT_ARRAY:
return None
value = self.element.document.get_float_array(self.offset, self.length)
if len(value) != self.length:
return None
return value
 
def get_file(self):
if self.type != self.TYPE_FILE:
return None
value = self.element.document.get_file(self.offset, self.length)
return value
 
def get_id(self):
if self.type != self.TYPE_ID:
return None
id = self.element.document.get_id_string(self.offset)
return id
 
def get_id_ref(self):
if self.type != self.TYPE_ID_REF:
return None
id = self.element.document.get_id_string(self.offset)
element = Element(self.element.document)
return [id, element]
 
def dump(self, f, depth):
pass
#print('  ' * depth + 'Attribute:' + 'name:{0} type:{1}'.format(self.name, self.type), end='\n', file=f)
 
class Element(object):
HEADER_FMT = ENDIANNESS + 'iiiiiii'
SIZE = struct.calcsize(HEADER_FMT)
 
TAG_NAME = 0
ATTR_NUM = 1
PARENT = 2
PREV = 3
NEXT = 4
FIRST_CHILD = 5
LAST_CHILD = 6
 
def __init__(self, document):
self.document = document
self.start = None
self.name = None
self.num_attributes = None
self.parent = None
self.prev = None
self.next = None
self.first_child = None
self.last_child = None
 
def load(self, f):
self.start = f.tell()
self.name, self.num_attributes, self.parent, self.prev, self.next, self.first_child, self.last_child = struct.unpack(self.HEADER_FMT, f.read(self.SIZE))
return True
 
def get_name(self):
return self.document.get_string(self.name)
 
def get_attribute(self, index):
if index < 0 or index >= self.num_attributes:
return None
offset = self.start + Element.SIZE + index * Attribute.SIZE
if not is_valid_attribute(self.document, offset):
return None
attribute = Attribute(self)
f = BytesIO(self.document.tree_bin)
f.seek(offset)
attribute.load(f)
return attribute
 
def get_parent(self):
if not is_valid_element(self.document, self.parent):
return None
element = Element(self.document)
f = BytesIO(self.document.tree_bin)
f.seek(parent)
element.load(f)
return element
 
def get_first_child(self):
if not is_valid_element(self.document, self.first_child):
return None
element = Element(self.document)
f = BytesIO(self.document.tree_bin)
f.seek(self.first_child)
element.load(f)
return element
 
def get_last_child(self):
if not is_valid_element(self.document, self.last_child):
return None
element = Element(self.document)
f = BytesIO(self.document.tree_bin)
f.seek(self.last_child)
element.load(f)
return element
 
def get_prev_sibling(self):
if not is_valid_element(self.document, self.prev):
return None
element = Element(self.document)
f = BytesIO(self.document.tree_bin)
f.seek(self.prev)
element.load(f)
return element
 
def get_next_sibling(self):
if not is_valid_element(self.document, self.next):
return None
element = Element(self.document)
f = BytesIO(self.document.tree_bin)
f.seek(self.next)
element.load(f)
return element
 
def dump(self, f, depth):
write_indent(f, depth)
name = self.get_name()
write_raw(f, '<' + name)
for i in range(self.num_attributes):
attribute = self.get_attribute(i)
if attribute is None:
return False
write_raw(f, ' {0}='.format(attribute.get_name()))
if attribute.type == Attribute.TYPE_NONE:
write_raw(f, '\"null\"')
elif attribute.type == Attribute.TYPE_INT:
write_raw(f, '\"{0}\"'.format(attribute.get_int()))
elif attribute.type == Attribute.TYPE_FLOAT:
write_raw(f, '\"{0:3.6}\"'.format(attribute.get_float()))
elif attribute.type == Attribute.TYPE_STRING:
write_raw(f, '\"{0}\"'.format(attribute.get_string()))
elif attribute.type == Attribute.TYPE_INT_ARRAY:
write_raw(f, '\"')
array = attribute.get_int_array()
array_length = len(array)
for j in range(array_length):
write_raw(f, '{0}'.format(array[j]))
if j + 1 < array_length:
write_raw(f, ',')
write_raw(f, '\"')
elif attribute.type == Attribute.TYPE_FLOAT_ARRAY:
write_raw(f, '\"')
array = attribute.get_float_array()
array_length = len(array)
for j in range(array_length):
write_raw(f, '{0:3.6}'.format(array[j]))
if j + 1 < array_length:
write_raw(f, ',')
write_raw(f, '\"')
elif attribute.type == Attribute.TYPE_FILE:
file_name = '{0}_0x{1:08X}.bin'.format(self.document.file_prefix, attribute.offset)
file_data = attribute.get_file()
with open(file_name, 'wb') as of:
of.write(file_data)
write_raw(f, '\"{0}\"'.format(file_name))
elif attribute.type == Attribute.TYPE_ID:
write_raw(f, '\"{0}\"'.format(attribute.get_id()))
elif attribute.type == Attribute.TYPE_ID_REF:
id_entity = attribute.get_id_ref()
write_raw(f, '\"{0}\"'.format(id_entity[0]))
child_element = self.get_first_child()
if not child_element is None:
write_raw(f, '>\n')
while not child_element is None:
child_element.dump(f, depth + 1)
child_element = child_element.get_next_sibling()
write_indent(f, depth)
write_raw(f, '</' + name + '>\n')
else:
write_raw(f, ' />\n')
 
def is_valid_element(document, offset):
if offset < 0 or offset + Element.SIZE > document.tree_size:
return False
element = Element(document)
f = BytesIO(document.tree_bin)
f.seek(offset)
element.load(f)
if element.num_attributes < 0 or offset + Element.SIZE + element.num_attributes * Attribute.SIZE > document.tree_size:
return False
return True
 
def is_valid_attribute(document, offset):
if offset < 0 or offset + Attribute.SIZE > document.tree_size:
return False
return True
 
class Document(object):
HEADER_FMT = ENDIANNESS + '4siiiiiiiiiiiii8x'
HEADER_SIZE = struct.calcsize(HEADER_FMT)
 
def __init__(self, file_prefix=''):
self.file_prefix = file_prefix
self.magic = None
self.version = None
self.tree_offset = None
self.tree_size = None
self.id_table_offset = None
self.id_table_size = None
self.string_table_offset = None
self.string_table_size = None
self.int_array_table_offset = None
self.int_array_table_size = None
self.float_array_table_offset = None
self.float_array_table_size = None
self.file_table_offset = None
self.file_table_size = None
self.tree_bin = None
self.id_table_bin = None
self.string_table_bin = None
self.int_array_table_bin = None
self.float_array_table_bin = None
self.file_table_bin = None
self.root = None
 
def get_document_element(self):
if not is_valid_element(self, 0):
return None
element = Element(self)
f = BytesIO(self.tree_bin)
element.load(f)
return element
 
def get_id_string(self, offset):
if offset < 0 or offset >= self.id_table_size:
return None
f = BytesIO(self.id_table_bin)
f.seek(offset)
entity_offset, = struct.unpack(INT_FMT, f.read(struct.calcsize(INT_FMT)))
return read_cstring(f)
 
def get_string(self, offset):
if offset < 0 or offset >= self.string_table_size:
return None
f = BytesIO(self.string_table_bin)
f.seek(offset)
return read_cstring(f)
 
def get_int_array(self, offset, length):
if offset < 0 or (offset + length) * struct.calcsize(INT_FMT) > self.int_array_table_size:
return None
f = BytesIO(self.int_array_table_bin)
f.seek(offset * struct.calcsize(INT_FMT))
array = []
for i in range(length):
value, = struct.unpack(INT_FMT, f.read(struct.calcsize(INT_FMT)))
array.append(value)
return array
 
def get_float_array(self, offset, length):
if offset < 0 or (offset + length) * struct.calcsize(FLOAT_FMT) > self.float_array_table_size:
return None
f = BytesIO(self.float_array_table_bin)
f.seek(offset * struct.calcsize(FLOAT_FMT))
array = []
for i in range(length):
value, = struct.unpack(FLOAT_FMT, f.read(struct.calcsize(FLOAT_FMT)))
array.append(value)
return array
 
def get_file(self, offset, length):
if offset < 0 or offset + length > self.file_table_size:
return None
return self.file_table_bin[offset:offset + length]
 
def load(self, f):
self.magic, self.version, self.tree_offset, self.tree_size, self.id_table_offset, self.id_table_size, self.string_table_offset, self.string_table_size, self.int_array_table_offset, self.int_array_table_size, self.float_array_table_offset, self.float_array_table_size, self.file_table_offset, self.file_table_size = struct.unpack(self.HEADER_FMT, f.read(self.HEADER_SIZE))
f.seek(self.tree_offset)
self.tree_bin = f.read(self.tree_size)
f.seek(self.id_table_offset)
self.id_table_bin = f.read(self.id_table_size)
f.seek(self.string_table_offset)
self.string_table_bin = f.read(self.string_table_size)
f.seek(self.int_array_table_offset)
self.int_array_table_bin = f.read(self.int_array_table_size)
f.seek(self.float_array_table_offset)
self.float_array_table_bin = f.read(self.float_array_table_size)
f.seek(self.file_table_offset)
self.file_table_bin = f.read(self.file_table_size)
self.root = self.get_document_element()
return True
 
def check(self, f):
return check_file_magic(f, 'CXML')
 
def dump(self, f=sys.stdout, depth=0):
if self.root is None:
return
self.root.dump(f, depth)
 
if len(sys.argv) < 3:
print('error: insufficient options specified')
sys.exit()
 
cxml_file_path = sys.argv[1]
if not os.path.isfile(cxml_file_path):
print('error: invalid cxml file specified')
sys.exit()
xml_file_path = sys.argv[2]
if os.path.exists(xml_file_path) and not os.path.isfile(xml_file_path):
print('error: invalid xml file specified')
sys.exit()
 
cxml_file_base = os.path.splitext(cxml_file_path)[0]
document = Document(cxml_file_base)
with open(cxml_file_path, 'rb') as f:
#if not document.check(f):
# print 'error: invalid CXML file format'
# sys.exit()
document.load(f)
 
with open(xml_file_path, 'wb') as f:
write_raw(f, '<?xml version="1.0" encoding="utf-8"?>\n')
document.dump(f)
</syntaxhighlight>}}
</div>

Latest revision as of 06:01, 15 April 2023

Other sony consoles[edit source]

  • In PSvita the RCO's uses a cxml structure, with a header size of 0x50 and the magic identifyer RCOF
    • RCOF header is 0x10 bytes bigger than any cxml header in PS3 (this means there are 2 new tables added for the new cxml variant used in vita rco's). And endianess changed because PSVita processor arquitecture
    • Some of the extracted files (after cropped and zlib decompressed) are another variant of a cxml structure with identifyer RCSF
    • More info about this cxml variants in PSVita wiki
Retrieved from "http://www.psdevwiki.com/ps3/index.php?title=Talk:CXML_Containers&oldid=70150"