Optimize overall memory usage

* Fixes in otbm loader
* Rework BinaryTree

src/framework/CMakeLists.txt

@@ -33,6 +33,7 @@ set(framework_SOURCES ${framework_SOURCES}
     ${CMAKE_CURRENT_LIST_DIR}/util/rsa.cpp
     ${CMAKE_CURRENT_LIST_DIR}/util/rsa.h
     ${CMAKE_CURRENT_LIST_DIR}/util/size.h
+    ${CMAKE_CURRENT_LIST_DIR}/util/attribstorage.h
 
     # stdext
     ${CMAKE_CURRENT_LIST_DIR}/stdext/cast.h

src/framework/core/binarytree.cpp

@@ -23,21 +23,54 @@
 #include "binarytree.h"
 #include "filestream.h"
 
-void BinaryTree::unserialize(const FileStreamPtr& fin)
+BinaryTree::BinaryTree(const FileStreamPtr& fin) :
+    m_fin(fin), m_pos(0xFFFFFFFF)
+{
+    m_startPos = fin->tell();
+}
+
+BinaryTree::~BinaryTree()
+{
+}
+
+void BinaryTree::skipNodes()
 {
     while(true) {
-        uint8 byte = fin->getU8();
+        uint8 byte = m_fin->getU8();
         switch(byte) {
             case BINARYTREE_NODE_START: {
-                BinaryTreePtr node(new BinaryTree(shared_from_this()));
-                m_children.push_back(node);
-                node->unserialize(fin);
+                skipNodes();
                 break;
             }
             case BINARYTREE_NODE_END:
                 return;
             case BINARYTREE_ESCAPE_CHAR:
-                m_buffer.add(fin->getU8());
+                m_fin->getU8();
+                break;
+            default:
+                break;
+        }
+    }
+}
+
+void BinaryTree::unserialize()
+{
+    if(m_pos != 0xFFFFFFFF)
+        return;
+    m_pos = 0;
+
+    m_fin->seek(m_startPos);
+    while(true) {
+        uint8 byte = m_fin->getU8();
+        switch(byte) {
+            case BINARYTREE_NODE_START: {
+                skipNodes();
+                break;
+            }
+            case BINARYTREE_NODE_END:
+                return;
+            case BINARYTREE_ESCAPE_CHAR:
+                m_buffer.add(m_fin->getU8());
                 break;
             default:
                 m_buffer.add(byte);
@@ -46,13 +79,33 @@ void BinaryTree::unserialize(const FileStreamPtr& fin)
     }
 }
 
-void BinaryTree::serialize(const FileStreamPtr& fin)
+BinaryTreeVec BinaryTree::getChildren()
 {
-
+    BinaryTreeVec children;
+    m_fin->seek(m_startPos);
+    while(true) {
+        uint8 byte = m_fin->getU8();
+        switch(byte) {
+            case BINARYTREE_NODE_START: {
+                BinaryTreePtr node(new BinaryTree(m_fin));
+                children.push_back(node);
+                node->skipNodes();
+                break;
+            }
+            case BINARYTREE_NODE_END:
+                return children;
+            case BINARYTREE_ESCAPE_CHAR:
+                m_fin->getU8();
+                break;
+            default:
+                break;
+        }
+    }
 }
 
 void BinaryTree::seek(uint pos)
 {
+    unserialize();
     if(pos > m_buffer.size())
         stdext::throw_exception("BinaryTree: seek failed");
     m_pos = pos;
@@ -60,6 +113,7 @@ void BinaryTree::seek(uint pos)
 
 uint8 BinaryTree::getU8()
 {
+    unserialize();
     if(m_pos+1 > m_buffer.size())
         stdext::throw_exception("BinaryTree: getU8 failed");
     uint8 v = m_buffer[m_pos];
@@ -69,6 +123,7 @@ uint8 BinaryTree::getU8()
 
 uint16 BinaryTree::getU16()
 {
+    unserialize();
     if(m_pos+2 > m_buffer.size())
         stdext::throw_exception("BinaryTree: getU16 failed");
     uint16 v = stdext::readLE16(&m_buffer[m_pos]);
@@ -78,6 +133,7 @@ uint16 BinaryTree::getU16()
 
 uint32 BinaryTree::getU32()
 {
+    unserialize();
     if(m_pos+4 > m_buffer.size())
         stdext::throw_exception("BinaryTree: getU32 failed");
     uint32 v = stdext::readLE32(&m_buffer[m_pos]);
@@ -87,6 +143,7 @@ uint32 BinaryTree::getU32()
 
 uint64 BinaryTree::getU64()
 {
+    unserialize();
     if(m_pos+8 > m_buffer.size())
         stdext::throw_exception("BinaryTree: getU64 failed");
     uint64 v = stdext::readLE64(&m_buffer[m_pos]);
@@ -96,10 +153,8 @@ uint64 BinaryTree::getU64()
 
 std::string BinaryTree::getString()
 {
+    unserialize();
     uint16 len = getU16();
-    if(len == 0 || len > 8192)
-        stdext::throw_exception("BinaryTree: getString failed: invalid or too large string length");
-
     if(m_pos+len > m_buffer.size())
         stdext::throw_exception("BinaryTree: getString failed: string length exceeded buffer size.");
 

src/framework/core/binarytree.h

@@ -32,18 +32,16 @@ enum {
     BINARYTREE_NODE_END = 0xFF
 };
 
-class BinaryTree : public std::enable_shared_from_this<BinaryTree>
+class BinaryTree
 {
 public:
-    BinaryTree(const BinaryTreePtr& parent = nullptr) : m_pos(0), m_parent(parent) { }
-
-    void unserialize(const FileStreamPtr& fin);
-    void serialize(const FileStreamPtr& fin);
+    BinaryTree(const FileStreamPtr& fin);
+    ~BinaryTree();
 
     void seek(uint pos);
     void skip(uint len) { seek(tell() + len); }
     uint tell() { return m_pos; }
-    uint size() { return m_buffer.size(); }
+    uint size() { unserialize(); return m_buffer.size(); }
 
     uint8 getU8();
     uint16 getU16();
@@ -51,16 +49,17 @@ public:
     uint64 getU64();
     std::string getString();
 
-    BinaryTreeVec getChildren() { return m_children; }
-    BinaryTreePtr getParent() { return m_parent.lock(); }
-    bool canRead() { return m_pos < m_buffer.size(); }
+    BinaryTreeVec getChildren();
+    bool canRead() { unserialize(); return m_pos < m_buffer.size(); }
 
 private:
-    uint m_pos;
+    void unserialize();
+    void skipNodes();
 
-    BinaryTreeVec m_children;
-    BinaryTreeWeakPtr m_parent;
+    FileStreamPtr m_fin;
     DataBuffer<uint8> m_buffer;
+    uint m_pos;
+    uint m_startPos;
 };
 
 #endif

src/framework/core/filestream.cpp

@@ -143,7 +143,7 @@ void FileStream::skip(uint len)
     seek(tell() + len);
 }
 
-int FileStream::size()
+uint FileStream::size()
 {
     if(!m_caching)
         return PHYSFS_fileLength(m_fileHandle);
@@ -151,7 +151,7 @@ int FileStream::size()
         return m_data.size();
 }
 
-int FileStream::tell()
+uint FileStream::tell()
 {
     if(!m_caching)
         return PHYSFS_tell(m_fileHandle);
@@ -249,13 +249,11 @@ std::string FileStream::getString()
 
 BinaryTreePtr FileStream::getBinaryTree()
 {
-    BinaryTreePtr root = BinaryTreePtr(new BinaryTree);
     uint8 byte = getU8();
-    if(byte == BINARYTREE_NODE_START)
-        root->unserialize(asFileStream());
-    else
+    if(byte != BINARYTREE_NODE_START)
         stdext::throw_exception(stdext::format("failed to read node start (getBinaryTree): %d", byte));
-    return root;
+
+    return BinaryTreePtr(new BinaryTree(asFileStream()));
 }
 
 void FileStream::addU8(uint8 v)

src/framework/core/filestream.h

@@ -43,8 +43,8 @@ public:
     int read(void *buffer, uint size, uint nmemb = 1);
     void seek(uint pos);
     void skip(uint len);
-    int size();
-    int tell();
+    uint size();
+    uint tell();
     std::string name() { return m_name; }
 
     uint8 getU8();

src/framework/luaengine/luainterface.cpp

@@ -721,7 +721,10 @@ void LuaInterface::error()
 
 int LuaInterface::ref()
 {
-    return luaL_ref(L, LUA_REGISTRYINDEX);
+    int ref = luaL_ref(L, LUA_REGISTRYINDEX);
+    assert(ref != LUA_NOREF);
+    assert(ref < 2147483647);
+    return ref;
 }
 
 int LuaInterface::weakRef()

src/framework/luaengine/luaobject.h

@@ -27,6 +27,7 @@
 
 /// LuaObject, all script-able classes have it as base
 // @bindclass
+#pragma pack(push,1) // disable memory alignment
 class LuaObject : public std::enable_shared_from_this<LuaObject>
 {
 public:
@@ -88,6 +89,7 @@ private:
     int m_fieldsTableRef;
     int m_metatableRef;
 };
+#pragma pack(pop)
 
 #include "luainterface.h"
 

src/framework/util/attribstorage.h

@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2010-2012 OTClient <https://github.com/edubart/otclient>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef ATTRIBSTORAGE_H
+#define ATTRIBSTORAGE_H
+
+#include "../stdext/types.h"
+#include <tuple>
+#include <boost/any.hpp>
+
+
+#pragma pack(push,1) // disable memory alignment
+
+// this class was designed to use less memory as possible
+class AttribStorage {
+public:
+    AttribStorage() : m_attribs(nullptr), m_size(0) { }
+    ~AttribStorage() { if(m_attribs) delete[] m_attribs; }
+
+    template<typename T>
+    void set(uint8 id, T value) {
+        bool done = false;
+        for(int i=0;i<m_size;++i) {
+            if(std::get<0>(m_attribs[i]) == id) {
+                std::get<1>(m_attribs[i]) = value;
+                done = true;
+                break;
+            }
+        }
+        if(!done) {
+            auto attribs = new std::tuple<uint8, boost::any>[m_size+1];
+            if(m_size > 0) {
+                for(int i=0;i<m_size;++i)
+                    attribs[i] = m_attribs[i];
+                delete[] m_attribs;
+            }
+            m_attribs = attribs;
+            m_attribs[m_size++] = std::make_tuple(id, value);
+        }
+    }
+
+    template<typename T>
+    T get(uint8 id) const {
+        for(int i=0;i<m_size;++i)
+            if(std::get<0>(m_attribs[i]) == id)
+                return boost::any_cast<T>(std::get<1>(m_attribs[i]));
+        return T();
+    }
+
+    bool has(uint8 id) const {
+        for(int i=0;i<m_size;++i)
+            if(std::get<0>(m_attribs[i]) == id)
+                return true;
+        return false;
+    }
+private:
+    std::tuple<uint8, boost::any>* m_attribs;
+    uint8 m_size;
+};
+#pragma pack(pop)
+
+#endif