New cbfstool. Works without mmap or fork/exec and

supports fixed location files. Some parts are salvaged
from the pre-commit version (esp. stage and payload creation),
others are completely rewritten (eg. the main loop that handles
file addition)

Also adapt newconfig (we don't need cbfs/tools anymore) and fix
some minor issues in the cbfstool-README.

Signed-off-by: Patrick Georgi <patrick.georgi@coresystems.de>
Acked-by: Stefan Reinauer <stepan@coresystems.de>


git-svn-id: svn://svn.coreboot.org/coreboot/trunk@4630 2b7e53f0-3cfb-0310-b3e9-8179ed1497e1
This commit is contained in:
Patrick Georgi
2009-09-14 13:29:27 +00:00
parent c8d4a05f8f
commit b7b56dd8fb
77 changed files with 656 additions and 2445 deletions

View File

@@ -0,0 +1,54 @@
// BinTree.h
#include "../LZInWindow.h"
#include "../IMatchFinder.h"
namespace BT_NAMESPACE {
typedef UInt32 CIndex;
const UInt32 kMaxValForNormalize = (UInt32(1) << 31) - 1;
class CMatchFinder:
public IMatchFinder,
public CLZInWindow,
public CMyUnknownImp,
public IMatchFinderSetNumPasses
{
UInt32 _cyclicBufferPos;
UInt32 _cyclicBufferSize; // it must be historySize + 1
UInt32 _matchMaxLen;
CIndex *_hash;
CIndex *_son;
UInt32 _hashMask;
UInt32 _cutValue;
UInt32 _hashSizeSum;
void Normalize();
void FreeThisClassMemory();
void FreeMemory();
MY_UNKNOWN_IMP
STDMETHOD(SetStream)(ISequentialInStream *inStream);
STDMETHOD_(void, ReleaseStream)();
STDMETHOD(Init)();
HRESULT MovePos();
STDMETHOD_(Byte, GetIndexByte)(Int32 index);
STDMETHOD_(UInt32, GetMatchLen)(Int32 index, UInt32 back, UInt32 limit);
STDMETHOD_(UInt32, GetNumAvailableBytes)();
STDMETHOD_(const Byte *, GetPointerToCurrentPos)();
STDMETHOD_(Int32, NeedChangeBufferPos)(UInt32 numCheckBytes);
STDMETHOD_(void, ChangeBufferPos)();
STDMETHOD(Create)(UInt32 historySize, UInt32 keepAddBufferBefore,
UInt32 matchMaxLen, UInt32 keepAddBufferAfter);
STDMETHOD(GetMatches)(UInt32 *distances);
STDMETHOD(Skip)(UInt32 num);
public:
CMatchFinder();
virtual ~CMatchFinder();
virtual void SetNumPasses(UInt32 numPasses) { _cutValue = numPasses; }
};
}

View File

@@ -0,0 +1,12 @@
// BinTree2.h
#ifndef __BINTREE2_H
#define __BINTREE2_H
#define BT_NAMESPACE NBT2
#include "BinTreeMain.h"
#undef BT_NAMESPACE
#endif

View File

@@ -0,0 +1,16 @@
// BinTree3.h
#ifndef __BINTREE3_H
#define __BINTREE3_H
#define BT_NAMESPACE NBT3
#define HASH_ARRAY_2
#include "BinTreeMain.h"
#undef HASH_ARRAY_2
#undef BT_NAMESPACE
#endif

View File

@@ -0,0 +1,18 @@
// BinTree4.h
#ifndef __BINTREE4_H
#define __BINTREE4_H
#define BT_NAMESPACE NBT4
#define HASH_ARRAY_2
#define HASH_ARRAY_3
#include "BinTreeMain.h"
#undef HASH_ARRAY_2
#undef HASH_ARRAY_3
#undef BT_NAMESPACE
#endif

View File

@@ -0,0 +1,531 @@
// BinTreeMain.h
#include "../../../../Common/Defs.h"
#include "../../../../Common/CRC.h"
#include "../../../../Common/Alloc.h"
#include "BinTree.h"
// #include <xmmintrin.h>
// It's for prefetch
// But prefetch doesn't give big gain in K8.
namespace BT_NAMESPACE {
#ifdef HASH_ARRAY_2
static const UInt32 kHash2Size = 1 << 10;
#define kNumHashDirectBytes 0
#ifdef HASH_ARRAY_3
static const UInt32 kNumHashBytes = 4;
static const UInt32 kHash3Size = 1 << 16;
#else
static const UInt32 kNumHashBytes = 3;
#endif
static const UInt32 kHashSize = 0;
static const UInt32 kMinMatchCheck = kNumHashBytes;
static const UInt32 kStartMaxLen = 1;
#else
#ifdef HASH_ZIP
#define kNumHashDirectBytes 0
static const UInt32 kNumHashBytes = 3;
static const UInt32 kHashSize = 1 << 16;
static const UInt32 kMinMatchCheck = kNumHashBytes;
static const UInt32 kStartMaxLen = 1;
#else
#define kNumHashDirectBytes 2
static const UInt32 kNumHashBytes = 2;
static const UInt32 kHashSize = 1 << (8 * kNumHashBytes);
static const UInt32 kMinMatchCheck = kNumHashBytes + 1;
static const UInt32 kStartMaxLen = 1;
#endif
#endif
#ifdef HASH_ARRAY_2
#ifdef HASH_ARRAY_3
static const UInt32 kHash3Offset = kHash2Size;
#endif
#endif
static const UInt32 kFixHashSize = 0
#ifdef HASH_ARRAY_2
+ kHash2Size
#ifdef HASH_ARRAY_3
+ kHash3Size
#endif
#endif
;
CMatchFinder::CMatchFinder():
_hash(0)
{
}
void CMatchFinder::FreeThisClassMemory()
{
BigFree(_hash);
_hash = 0;
}
void CMatchFinder::FreeMemory()
{
FreeThisClassMemory();
CLZInWindow::Free();
}
CMatchFinder::~CMatchFinder()
{
FreeMemory();
}
STDMETHODIMP CMatchFinder::Create(UInt32 historySize, UInt32 keepAddBufferBefore,
UInt32 matchMaxLen, UInt32 keepAddBufferAfter)
{
if (historySize > kMaxValForNormalize - 256)
{
FreeMemory();
return E_INVALIDARG;
}
_cutValue =
#ifdef _HASH_CHAIN
8 + (matchMaxLen >> 2);
#else
16 + (matchMaxLen >> 1);
#endif
UInt32 sizeReserv = (historySize + keepAddBufferBefore +
matchMaxLen + keepAddBufferAfter) / 2 + 256;
if (CLZInWindow::Create(historySize + keepAddBufferBefore,
matchMaxLen + keepAddBufferAfter, sizeReserv))
{
_matchMaxLen = matchMaxLen;
UInt32 newCyclicBufferSize = historySize + 1;
if (_hash != 0 && newCyclicBufferSize == _cyclicBufferSize)
return S_OK;
FreeThisClassMemory();
_cyclicBufferSize = newCyclicBufferSize; // don't change it
UInt32 hs = kHashSize;
#ifdef HASH_ARRAY_2
hs = historySize - 1;
hs |= (hs >> 1);
hs |= (hs >> 2);
hs |= (hs >> 4);
hs |= (hs >> 8);
hs >>= 1;
hs |= 0xFFFF;
if (hs > (1 << 24))
{
#ifdef HASH_ARRAY_3
hs >>= 1;
#else
hs = (1 << 24) - 1;
#endif
}
_hashMask = hs;
hs++;
#endif
_hashSizeSum = hs + kFixHashSize;
UInt32 numItems = _hashSizeSum + _cyclicBufferSize
#ifndef _HASH_CHAIN
* 2
#endif
;
size_t sizeInBytes = (size_t)numItems * sizeof(CIndex);
if (sizeInBytes / sizeof(CIndex) != numItems)
return E_OUTOFMEMORY;
_hash = (CIndex *)BigAlloc(sizeInBytes);
_son = _hash + _hashSizeSum;
if (_hash != 0)
return S_OK;
}
FreeMemory();
return E_OUTOFMEMORY;
}
static const UInt32 kEmptyHashValue = 0;
STDMETHODIMP CMatchFinder::SetStream(ISequentialInStream *stream)
{
CLZInWindow::SetStream(stream);
return S_OK;
}
STDMETHODIMP CMatchFinder::Init()
{
RINOK(CLZInWindow::Init());
for(UInt32 i = 0; i < _hashSizeSum; i++)
_hash[i] = kEmptyHashValue;
_cyclicBufferPos = 0;
ReduceOffsets(-1);
return S_OK;
}
STDMETHODIMP_(void) CMatchFinder::ReleaseStream()
{
// ReleaseStream();
}
#ifdef HASH_ARRAY_2
#ifdef HASH_ARRAY_3
#define HASH_CALC { \
UInt32 temp = CCRC::Table[cur[0]] ^ cur[1]; \
hash2Value = temp & (kHash2Size - 1); \
hash3Value = (temp ^ (UInt32(cur[2]) << 8)) & (kHash3Size - 1); \
hashValue = (temp ^ (UInt32(cur[2]) << 8) ^ (CCRC::Table[cur[3]] << 5)) & _hashMask; }
#else // no HASH_ARRAY_3
#define HASH_CALC { \
UInt32 temp = CCRC::Table[cur[0]] ^ cur[1]; \
hash2Value = temp & (kHash2Size - 1); \
hashValue = (temp ^ (UInt32(cur[2]) << 8)) & _hashMask; }
#endif // HASH_ARRAY_3
#else // no HASH_ARRAY_2
#ifdef HASH_ZIP
inline UInt32 Hash(const Byte *pointer)
{
return ((UInt32(pointer[0]) << 8) ^ CCRC::Table[pointer[1]] ^ pointer[2]) & (kHashSize - 1);
}
#else // no HASH_ZIP
inline UInt32 Hash(const Byte *pointer)
{
return pointer[0] ^ (UInt32(pointer[1]) << 8);
}
#endif // HASH_ZIP
#endif // HASH_ARRAY_2
STDMETHODIMP CMatchFinder::GetMatches(UInt32 *distances)
{
UInt32 lenLimit;
if (_pos + _matchMaxLen <= _streamPos)
lenLimit = _matchMaxLen;
else
{
lenLimit = _streamPos - _pos;
if(lenLimit < kMinMatchCheck)
{
distances[0] = 0;
return MovePos();
}
}
int offset = 1;
UInt32 matchMinPos = (_pos > _cyclicBufferSize) ? (_pos - _cyclicBufferSize) : 0;
const Byte *cur = _buffer + _pos;
UInt32 maxLen = kStartMaxLen; // to avoid items for len < hashSize;
#ifdef HASH_ARRAY_2
UInt32 hash2Value;
#ifdef HASH_ARRAY_3
UInt32 hash3Value;
#endif
UInt32 hashValue;
HASH_CALC;
#else
UInt32 hashValue = Hash(cur);
#endif
UInt32 curMatch = _hash[kFixHashSize + hashValue];
#ifdef HASH_ARRAY_2
UInt32 curMatch2 = _hash[hash2Value];
#ifdef HASH_ARRAY_3
UInt32 curMatch3 = _hash[kHash3Offset + hash3Value];
#endif
_hash[hash2Value] = _pos;
if(curMatch2 > matchMinPos)
if (_buffer[curMatch2] == cur[0])
{
distances[offset++] = maxLen = 2;
distances[offset++] = _pos - curMatch2 - 1;
}
#ifdef HASH_ARRAY_3
_hash[kHash3Offset + hash3Value] = _pos;
if(curMatch3 > matchMinPos)
if (_buffer[curMatch3] == cur[0])
{
if (curMatch3 == curMatch2)
offset -= 2;
distances[offset++] = maxLen = 3;
distances[offset++] = _pos - curMatch3 - 1;
curMatch2 = curMatch3;
}
#endif
if (offset != 1 && curMatch2 == curMatch)
{
offset -= 2;
maxLen = kStartMaxLen;
}
#endif
_hash[kFixHashSize + hashValue] = _pos;
CIndex *son = _son;
#ifdef _HASH_CHAIN
son[_cyclicBufferPos] = curMatch;
#else
CIndex *ptr0 = son + (_cyclicBufferPos << 1) + 1;
CIndex *ptr1 = son + (_cyclicBufferPos << 1);
UInt32 len0, len1;
len0 = len1 = kNumHashDirectBytes;
#endif
#if kNumHashDirectBytes != 0
if(curMatch > matchMinPos)
{
if (_buffer[curMatch + kNumHashDirectBytes] != cur[kNumHashDirectBytes])
{
distances[offset++] = maxLen = kNumHashDirectBytes;
distances[offset++] = _pos - curMatch - 1;
}
}
#endif
UInt32 count = _cutValue;
while(true)
{
if(curMatch <= matchMinPos || count-- == 0)
{
#ifndef _HASH_CHAIN
*ptr0 = *ptr1 = kEmptyHashValue;
#endif
break;
}
UInt32 delta = _pos - curMatch;
UInt32 cyclicPos = (delta <= _cyclicBufferPos) ?
(_cyclicBufferPos - delta):
(_cyclicBufferPos - delta + _cyclicBufferSize);
CIndex *pair = son +
#ifdef _HASH_CHAIN
cyclicPos;
#else
(cyclicPos << 1);
#endif
// _mm_prefetch((const char *)pair, _MM_HINT_T0);
const Byte *pb = _buffer + curMatch;
UInt32 len =
#ifdef _HASH_CHAIN
kNumHashDirectBytes;
if (pb[maxLen] == cur[maxLen])
#else
MyMin(len0, len1);
#endif
if (pb[len] == cur[len])
{
while(++len != lenLimit)
if (pb[len] != cur[len])
break;
if (maxLen < len)
{
distances[offset++] = maxLen = len;
distances[offset++] = delta - 1;
if (len == lenLimit)
{
#ifndef _HASH_CHAIN
*ptr1 = pair[0];
*ptr0 = pair[1];
#endif
break;
}
}
}
#ifdef _HASH_CHAIN
curMatch = *pair;
#else
if (pb[len] < cur[len])
{
*ptr1 = curMatch;
ptr1 = pair + 1;
curMatch = *ptr1;
len1 = len;
}
else
{
*ptr0 = curMatch;
ptr0 = pair;
curMatch = *ptr0;
len0 = len;
}
#endif
}
distances[0] = offset - 1;
if (++_cyclicBufferPos == _cyclicBufferSize)
_cyclicBufferPos = 0;
RINOK(CLZInWindow::MovePos());
if (_pos == kMaxValForNormalize)
Normalize();
return S_OK;
}
STDMETHODIMP CMatchFinder::Skip(UInt32 num)
{
do
{
#ifdef _HASH_CHAIN
if (_streamPos - _pos < kNumHashBytes)
{
RINOK(MovePos());
continue;
}
#else
UInt32 lenLimit;
if (_pos + _matchMaxLen <= _streamPos)
lenLimit = _matchMaxLen;
else
{
lenLimit = _streamPos - _pos;
if(lenLimit < kMinMatchCheck)
{
RINOK(MovePos());
continue;
}
}
UInt32 matchMinPos = (_pos > _cyclicBufferSize) ? (_pos - _cyclicBufferSize) : 0;
#endif
const Byte *cur = _buffer + _pos;
#ifdef HASH_ARRAY_2
UInt32 hash2Value;
#ifdef HASH_ARRAY_3
UInt32 hash3Value;
UInt32 hashValue;
HASH_CALC;
_hash[kHash3Offset + hash3Value] = _pos;
#else
UInt32 hashValue;
HASH_CALC;
#endif
_hash[hash2Value] = _pos;
#else
UInt32 hashValue = Hash(cur);
#endif
UInt32 curMatch = _hash[kFixHashSize + hashValue];
_hash[kFixHashSize + hashValue] = _pos;
#ifdef _HASH_CHAIN
_son[_cyclicBufferPos] = curMatch;
#else
CIndex *son = _son;
CIndex *ptr0 = son + (_cyclicBufferPos << 1) + 1;
CIndex *ptr1 = son + (_cyclicBufferPos << 1);
UInt32 len0, len1;
len0 = len1 = kNumHashDirectBytes;
UInt32 count = _cutValue;
while(true)
{
if(curMatch <= matchMinPos || count-- == 0)
{
*ptr0 = *ptr1 = kEmptyHashValue;
break;
}
UInt32 delta = _pos - curMatch;
UInt32 cyclicPos = (delta <= _cyclicBufferPos) ?
(_cyclicBufferPos - delta):
(_cyclicBufferPos - delta + _cyclicBufferSize);
CIndex *pair = son + (cyclicPos << 1);
// _mm_prefetch((const char *)pair, _MM_HINT_T0);
const Byte *pb = _buffer + curMatch;
UInt32 len = MyMin(len0, len1);
if (pb[len] == cur[len])
{
while(++len != lenLimit)
if (pb[len] != cur[len])
break;
if (len == lenLimit)
{
*ptr1 = pair[0];
*ptr0 = pair[1];
break;
}
}
if (pb[len] < cur[len])
{
*ptr1 = curMatch;
ptr1 = pair + 1;
curMatch = *ptr1;
len1 = len;
}
else
{
*ptr0 = curMatch;
ptr0 = pair;
curMatch = *ptr0;
len0 = len;
}
}
#endif
if (++_cyclicBufferPos == _cyclicBufferSize)
_cyclicBufferPos = 0;
RINOK(CLZInWindow::MovePos());
if (_pos == kMaxValForNormalize)
Normalize();
}
while(--num != 0);
return S_OK;
}
void CMatchFinder::Normalize()
{
UInt32 subValue = _pos - _cyclicBufferSize;
CIndex *items = _hash;
UInt32 numItems = (_hashSizeSum + _cyclicBufferSize
#ifndef _HASH_CHAIN
* 2
#endif
);
for (UInt32 i = 0; i < numItems; i++)
{
UInt32 value = items[i];
if (value <= subValue)
value = kEmptyHashValue;
else
value -= subValue;
items[i] = value;
}
ReduceOffsets(subValue);
}
HRESULT CMatchFinder::MovePos()
{
if (++_cyclicBufferPos == _cyclicBufferSize)
_cyclicBufferPos = 0;
RINOK(CLZInWindow::MovePos());
if (_pos == kMaxValForNormalize)
Normalize();
return S_OK;
}
STDMETHODIMP_(Byte) CMatchFinder::GetIndexByte(Int32 index)
{ return CLZInWindow::GetIndexByte(index); }
STDMETHODIMP_(UInt32) CMatchFinder::GetMatchLen(Int32 index,
UInt32 back, UInt32 limit)
{ return CLZInWindow::GetMatchLen(index, back, limit); }
STDMETHODIMP_(UInt32) CMatchFinder::GetNumAvailableBytes()
{ return CLZInWindow::GetNumAvailableBytes(); }
STDMETHODIMP_(const Byte *) CMatchFinder::GetPointerToCurrentPos()
{ return CLZInWindow::GetPointerToCurrentPos(); }
STDMETHODIMP_(Int32) CMatchFinder::NeedChangeBufferPos(UInt32 numCheckBytes)
{ return CLZInWindow::NeedMove(numCheckBytes) ? 1: 0; }
STDMETHODIMP_(void) CMatchFinder::ChangeBufferPos()
{ CLZInWindow::MoveBlock();}
#undef HASH_CALC
#undef kNumHashDirectBytes
}

View File

@@ -0,0 +1,19 @@
// HC4.h
#ifndef __HC4_H
#define __HC4_H
#define BT_NAMESPACE NHC4
#define HASH_ARRAY_2
#define HASH_ARRAY_3
#include "HCMain.h"
#undef HASH_ARRAY_2
#undef HASH_ARRAY_3
#undef BT_NAMESPACE
#endif

View File

@@ -0,0 +1,6 @@
// HCMain.h
#define _HASH_CHAIN
#include "../BinTree/BinTreeMain.h"
#undef _HASH_CHAIN

View File

@@ -0,0 +1,33 @@
// MatchFinders/IMatchFinder.h
#ifndef __IMATCHFINDER_H
#define __IMATCHFINDER_H
struct IInWindowStream: public IUnknown
{
STDMETHOD(SetStream)(ISequentialInStream *inStream) PURE;
STDMETHOD_(void, ReleaseStream)() PURE;
STDMETHOD(Init)() PURE;
STDMETHOD_(Byte, GetIndexByte)(Int32 index) PURE;
STDMETHOD_(UInt32, GetMatchLen)(Int32 index, UInt32 distance, UInt32 limit) PURE;
STDMETHOD_(UInt32, GetNumAvailableBytes)() PURE;
STDMETHOD_(const Byte *, GetPointerToCurrentPos)() PURE;
STDMETHOD_(Int32, NeedChangeBufferPos)(UInt32 numCheckBytes) PURE;
STDMETHOD_(void, ChangeBufferPos)() PURE;
};
struct IMatchFinder: public IInWindowStream
{
STDMETHOD(Create)(UInt32 historySize, UInt32 keepAddBufferBefore,
UInt32 matchMaxLen, UInt32 keepAddBufferAfter) PURE;
STDMETHOD(GetMatches)(UInt32 *distances) PURE;
STDMETHOD(Skip)(UInt32 num) PURE;
};
struct IMatchFinderSetNumPasses
{
virtual ~IMatchFinderSetNumPasses(){}
virtual void SetNumPasses(UInt32 numPasses) PURE;
};
#endif

View File

@@ -0,0 +1,105 @@
// LZInWindow.cpp
#include "StdAfx.h"
#include "LZInWindow.h"
#include "../../../Common/MyCom.h"
#include "../../../Common/Alloc.h"
void CLZInWindow::Free()
{
::BigFree(_bufferBase);
_bufferBase = 0;
}
bool CLZInWindow::Create(UInt32 keepSizeBefore, UInt32 keepSizeAfter, UInt32 keepSizeReserv)
{
_keepSizeBefore = keepSizeBefore;
_keepSizeAfter = keepSizeAfter;
UInt32 blockSize = keepSizeBefore + keepSizeAfter + keepSizeReserv;
if (_bufferBase == 0 || _blockSize != blockSize)
{
Free();
_blockSize = blockSize;
if (_blockSize != 0)
_bufferBase = (Byte *)::BigAlloc(_blockSize);
}
_pointerToLastSafePosition = _bufferBase + _blockSize - keepSizeAfter;
if (_blockSize == 0)
return true;
return (_bufferBase != 0);
}
void CLZInWindow::SetStream(ISequentialInStream *stream)
{
_stream = stream;
}
HRESULT CLZInWindow::Init()
{
_buffer = _bufferBase;
_pos = 0;
_streamPos = 0;
_streamEndWasReached = false;
return ReadBlock();
}
/*
void CLZInWindow::ReleaseStream()
{
_stream.Release();
}
*/
///////////////////////////////////////////
// ReadBlock
// In State:
// (_buffer + _streamPos) <= (_bufferBase + _blockSize)
// Out State:
// _posLimit <= _blockSize - _keepSizeAfter;
// if(_streamEndWasReached == false):
// _streamPos >= _pos + _keepSizeAfter
// _posLimit = _streamPos - _keepSizeAfter;
// else
//
HRESULT CLZInWindow::ReadBlock()
{
if(_streamEndWasReached)
return S_OK;
while(true)
{
UInt32 size = (UInt32)(_bufferBase - _buffer) + _blockSize - _streamPos;
if(size == 0)
return S_OK;
UInt32 numReadBytes;
RINOK(_stream->Read(_buffer + _streamPos, size, &numReadBytes));
if(numReadBytes == 0)
{
_posLimit = _streamPos;
const Byte *pointerToPostion = _buffer + _posLimit;
if(pointerToPostion > _pointerToLastSafePosition)
_posLimit = (UInt32)(_pointerToLastSafePosition - _buffer);
_streamEndWasReached = true;
return S_OK;
}
_streamPos += numReadBytes;
if(_streamPos >= _pos + _keepSizeAfter)
{
_posLimit = _streamPos - _keepSizeAfter;
return S_OK;
}
}
}
void CLZInWindow::MoveBlock()
{
UInt32 offset = (UInt32)(_buffer - _bufferBase) + _pos - _keepSizeBefore;
// we need one additional byte, since MovePos moves on 1 byte.
if (offset > 0)
offset--;
UInt32 numBytes = (UInt32)(_buffer - _bufferBase) + _streamPos - offset;
memmove(_bufferBase, _bufferBase + offset, numBytes);
_buffer -= offset;
}

View File

@@ -0,0 +1,87 @@
// LZInWindow.h
#ifndef __LZ_IN_WINDOW_H
#define __LZ_IN_WINDOW_H
#include "../../IStream.h"
class CLZInWindow
{
Byte *_bufferBase; // pointer to buffer with data
ISequentialInStream *_stream;
UInt32 _posLimit; // offset (from _buffer) when new block reading must be done
bool _streamEndWasReached; // if (true) then _streamPos shows real end of stream
const Byte *_pointerToLastSafePosition;
protected:
Byte *_buffer; // Pointer to virtual Buffer begin
UInt32 _blockSize; // Size of Allocated memory block
UInt32 _pos; // offset (from _buffer) of curent byte
UInt32 _keepSizeBefore; // how many BYTEs must be kept in buffer before _pos
UInt32 _keepSizeAfter; // how many BYTEs must be kept buffer after _pos
UInt32 _streamPos; // offset (from _buffer) of first not read byte from Stream
void MoveBlock();
HRESULT ReadBlock();
void Free();
public:
CLZInWindow(): _bufferBase(0) {}
virtual ~CLZInWindow() { Free(); }
// keepSizeBefore + keepSizeAfter + keepSizeReserv < 4G)
bool Create(UInt32 keepSizeBefore, UInt32 keepSizeAfter, UInt32 keepSizeReserv = (1<<17));
void SetStream(ISequentialInStream *stream);
HRESULT Init();
// void ReleaseStream();
Byte *GetBuffer() const { return _buffer; }
const Byte *GetPointerToCurrentPos() const { return _buffer + _pos; }
HRESULT MovePos()
{
_pos++;
if (_pos > _posLimit)
{
const Byte *pointerToPostion = _buffer + _pos;
if(pointerToPostion > _pointerToLastSafePosition)
MoveBlock();
return ReadBlock();
}
else
return S_OK;
}
Byte GetIndexByte(Int32 index) const { return _buffer[(size_t)_pos + index]; }
// index + limit have not to exceed _keepSizeAfter;
// -2G <= index < 2G
UInt32 GetMatchLen(Int32 index, UInt32 distance, UInt32 limit) const
{
if(_streamEndWasReached)
if ((_pos + index) + limit > _streamPos)
limit = _streamPos - (_pos + index);
distance++;
const Byte *pby = _buffer + (size_t)_pos + index;
UInt32 i;
for(i = 0; i < limit && pby[i] == pby[(size_t)i - distance]; i++);
return i;
}
UInt32 GetNumAvailableBytes() const { return _streamPos - _pos; }
void ReduceOffsets(Int32 subValue)
{
_buffer += subValue;
_posLimit -= subValue;
_pos -= subValue;
_streamPos -= subValue;
}
bool NeedMove(UInt32 numCheckBytes)
{
UInt32 reserv = _pointerToLastSafePosition - (_buffer + _pos);
return (reserv <= numCheckBytes);
}
};
#endif

View File

@@ -0,0 +1,6 @@
// StdAfx.h
#ifndef __STDAFX_H
#define __STDAFX_H
#endif

View File

@@ -0,0 +1,82 @@
// LZMA.h
#ifndef __LZMA_H
#define __LZMA_H
namespace NCompress {
namespace NLZMA {
const UInt32 kNumRepDistances = 4;
const int kNumStates = 12;
const Byte kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5};
const Byte kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10};
const Byte kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11};
const Byte kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11};
class CState
{
public:
Byte Index;
void Init() { Index = 0; }
void UpdateChar() { Index = kLiteralNextStates[Index]; }
void UpdateMatch() { Index = kMatchNextStates[Index]; }
void UpdateRep() { Index = kRepNextStates[Index]; }
void UpdateShortRep() { Index = kShortRepNextStates[Index]; }
bool IsCharState() const { return Index < 7; }
};
const int kNumPosSlotBits = 6;
const int kDicLogSizeMin = 0;
const int kDicLogSizeMax = 32;
const int kDistTableSizeMax = kDicLogSizeMax * 2;
const UInt32 kNumLenToPosStates = 4;
inline UInt32 GetLenToPosState(UInt32 len)
{
len -= 2;
if (len < kNumLenToPosStates)
return len;
return kNumLenToPosStates - 1;
}
namespace NLength {
const int kNumPosStatesBitsMax = 4;
const UInt32 kNumPosStatesMax = (1 << kNumPosStatesBitsMax);
const int kNumPosStatesBitsEncodingMax = 4;
const UInt32 kNumPosStatesEncodingMax = (1 << kNumPosStatesBitsEncodingMax);
const int kNumLowBits = 3;
const int kNumMidBits = 3;
const int kNumHighBits = 8;
const UInt32 kNumLowSymbols = 1 << kNumLowBits;
const UInt32 kNumMidSymbols = 1 << kNumMidBits;
const UInt32 kNumSymbolsTotal = kNumLowSymbols + kNumMidSymbols + (1 << kNumHighBits);
}
const UInt32 kMatchMinLen = 2;
const UInt32 kMatchMaxLen = kMatchMinLen + NLength::kNumSymbolsTotal - 1;
const int kNumAlignBits = 4;
const UInt32 kAlignTableSize = 1 << kNumAlignBits;
const UInt32 kAlignMask = (kAlignTableSize - 1);
const UInt32 kStartPosModelIndex = 4;
const UInt32 kEndPosModelIndex = 14;
const UInt32 kNumPosModels = kEndPosModelIndex - kStartPosModelIndex;
const UInt32 kNumFullDistances = 1 << (kEndPosModelIndex / 2);
const int kNumLitPosStatesBitsEncodingMax = 4;
const int kNumLitContextBitsMax = 8;
const int kNumMoveBits = 5;
}}
#endif

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,411 @@
// LZMA/Encoder.h
#ifndef __LZMA_ENCODER_H
#define __LZMA_ENCODER_H
#include "../../../Common/MyCom.h"
#include "../../../Common/Alloc.h"
#include "../../ICoder.h"
#include "../LZ/IMatchFinder.h"
#include "../RangeCoder/RangeCoderBitTree.h"
#include "LZMA.h"
namespace NCompress {
namespace NLZMA {
typedef NRangeCoder::CBitEncoder<kNumMoveBits> CMyBitEncoder;
class CBaseState
{
protected:
CState _state;
Byte _previousByte;
UInt32 _repDistances[kNumRepDistances];
void Init()
{
_state.Init();
_previousByte = 0;
for(UInt32 i = 0 ; i < kNumRepDistances; i++)
_repDistances[i] = 0;
}
};
struct COptimal
{
CState State;
bool Prev1IsChar;
bool Prev2;
UInt32 PosPrev2;
UInt32 BackPrev2;
UInt32 Price;
UInt32 PosPrev; // posNext;
UInt32 BackPrev;
UInt32 Backs[kNumRepDistances];
void MakeAsChar() { BackPrev = UInt32(-1); Prev1IsChar = false; }
void MakeAsShortRep() { BackPrev = 0; ; Prev1IsChar = false; }
bool IsShortRep() { return (BackPrev == 0); }
};
extern Byte g_FastPos[1 << 11];
inline UInt32 GetPosSlot(UInt32 pos)
{
if (pos < (1 << 11))
return g_FastPos[pos];
if (pos < (1 << 21))
return g_FastPos[pos >> 10] + 20;
return g_FastPos[pos >> 20] + 40;
}
inline UInt32 GetPosSlot2(UInt32 pos)
{
if (pos < (1 << 17))
return g_FastPos[pos >> 6] + 12;
if (pos < (1 << 27))
return g_FastPos[pos >> 16] + 32;
return g_FastPos[pos >> 26] + 52;
}
const UInt32 kIfinityPrice = 0xFFFFFFF;
const UInt32 kNumOpts = 1 << 12;
class CLiteralEncoder2
{
CMyBitEncoder _encoders[0x300];
public:
void Init()
{
for (int i = 0; i < 0x300; i++)
_encoders[i].Init();
}
void Encode(NRangeCoder::CEncoder *rangeEncoder, Byte symbol);
void EncodeMatched(NRangeCoder::CEncoder *rangeEncoder, Byte matchByte, Byte symbol);
UInt32 GetPrice(bool matchMode, Byte matchByte, Byte symbol) const;
};
class CLiteralEncoder
{
CLiteralEncoder2 *_coders;
int _numPrevBits;
int _numPosBits;
UInt32 _posMask;
public:
CLiteralEncoder(): _coders(0) {}
~CLiteralEncoder() { Free(); }
void Free()
{
MyFree(_coders);
_coders = 0;
}
bool Create(int numPosBits, int numPrevBits)
{
if (_coders == 0 || (numPosBits + numPrevBits) != (_numPrevBits + _numPosBits))
{
Free();
UInt32 numStates = 1 << (numPosBits + numPrevBits);
_coders = (CLiteralEncoder2 *)MyAlloc(numStates * sizeof(CLiteralEncoder2));
}
_numPosBits = numPosBits;
_posMask = (1 << numPosBits) - 1;
_numPrevBits = numPrevBits;
return (_coders != 0);
}
void Init()
{
UInt32 numStates = 1 << (_numPrevBits + _numPosBits);
for (UInt32 i = 0; i < numStates; i++)
_coders[i].Init();
}
CLiteralEncoder2 *GetSubCoder(UInt32 pos, Byte prevByte)
{ return &_coders[((pos & _posMask) << _numPrevBits) + (prevByte >> (8 - _numPrevBits))]; }
};
namespace NLength {
class CEncoder
{
CMyBitEncoder _choice;
CMyBitEncoder _choice2;
NRangeCoder::CBitTreeEncoder<kNumMoveBits, kNumLowBits> _lowCoder[kNumPosStatesEncodingMax];
NRangeCoder::CBitTreeEncoder<kNumMoveBits, kNumMidBits> _midCoder[kNumPosStatesEncodingMax];
NRangeCoder::CBitTreeEncoder<kNumMoveBits, kNumHighBits> _highCoder;
public:
void Init(UInt32 numPosStates);
void Encode(NRangeCoder::CEncoder *rangeEncoder, UInt32 symbol, UInt32 posState);
void SetPrices(UInt32 posState, UInt32 numSymbols, UInt32 *prices) const;
};
const UInt32 kNumSpecSymbols = kNumLowSymbols + kNumMidSymbols;
class CPriceTableEncoder: public CEncoder
{
UInt32 _prices[kNumPosStatesEncodingMax][kNumSymbolsTotal];
UInt32 _tableSize;
UInt32 _counters[kNumPosStatesEncodingMax];
public:
void SetTableSize(UInt32 tableSize) { _tableSize = tableSize; }
UInt32 GetPrice(UInt32 symbol, UInt32 posState) const { return _prices[posState][symbol]; }
void UpdateTable(UInt32 posState)
{
SetPrices(posState, _tableSize, _prices[posState]);
_counters[posState] = _tableSize;
}
void UpdateTables(UInt32 numPosStates)
{
for (UInt32 posState = 0; posState < numPosStates; posState++)
UpdateTable(posState);
}
void Encode(NRangeCoder::CEncoder *rangeEncoder, UInt32 symbol, UInt32 posState, bool updatePrice)
{
CEncoder::Encode(rangeEncoder, symbol, posState);
if (updatePrice)
if (--_counters[posState] == 0)
UpdateTable(posState);
}
};
}
class CEncoder :
public ICompressCoder,
public ICompressSetOutStream,
public ICompressSetCoderProperties,
public ICompressWriteCoderProperties,
public CBaseState,
public CMyUnknownImp
{
COptimal _optimum[kNumOpts];
CMyComPtr<IMatchFinder> _matchFinder; // test it
NRangeCoder::CEncoder _rangeEncoder;
CMyBitEncoder _isMatch[kNumStates][NLength::kNumPosStatesEncodingMax];
CMyBitEncoder _isRep[kNumStates];
CMyBitEncoder _isRepG0[kNumStates];
CMyBitEncoder _isRepG1[kNumStates];
CMyBitEncoder _isRepG2[kNumStates];
CMyBitEncoder _isRep0Long[kNumStates][NLength::kNumPosStatesEncodingMax];
NRangeCoder::CBitTreeEncoder<kNumMoveBits, kNumPosSlotBits> _posSlotEncoder[kNumLenToPosStates];
CMyBitEncoder _posEncoders[kNumFullDistances - kEndPosModelIndex];
NRangeCoder::CBitTreeEncoder<kNumMoveBits, kNumAlignBits> _posAlignEncoder;
NLength::CPriceTableEncoder _lenEncoder;
NLength::CPriceTableEncoder _repMatchLenEncoder;
CLiteralEncoder _literalEncoder;
UInt32 _matchDistances[kMatchMaxLen * 2 + 2 + 1];
bool _fastMode;
// bool _maxMode;
UInt32 _numFastBytes;
UInt32 _longestMatchLength;
UInt32 _numDistancePairs;
UInt32 _additionalOffset;
UInt32 _optimumEndIndex;
UInt32 _optimumCurrentIndex;
bool _longestMatchWasFound;
UInt32 _posSlotPrices[kNumLenToPosStates][kDistTableSizeMax];
UInt32 _distancesPrices[kNumLenToPosStates][kNumFullDistances];
UInt32 _alignPrices[kAlignTableSize];
UInt32 _alignPriceCount;
UInt32 _distTableSize;
UInt32 _posStateBits;
UInt32 _posStateMask;
UInt32 _numLiteralPosStateBits;
UInt32 _numLiteralContextBits;
UInt32 _dictionarySize;
UInt32 _dictionarySizePrev;
UInt32 _numFastBytesPrev;
UInt32 _matchPriceCount;
UInt64 nowPos64;
bool _finished;
ISequentialInStream *_inStream;
UInt32 _matchFinderCycles;
int _matchFinderIndex;
#ifdef COMPRESS_MF_MT
bool _multiThread;
#endif
bool _writeEndMark;
bool _needReleaseMFStream;
IMatchFinderSetNumPasses *setMfPasses;
void ReleaseMatchFinder()
{
setMfPasses = 0;
_matchFinder.Release();
}
HRESULT ReadMatchDistances(UInt32 &len, UInt32 &numDistancePairs);
HRESULT MovePos(UInt32 num);
UInt32 GetRepLen1Price(CState state, UInt32 posState) const
{
return _isRepG0[state.Index].GetPrice0() +
_isRep0Long[state.Index][posState].GetPrice0();
}
UInt32 GetPureRepPrice(UInt32 repIndex, CState state, UInt32 posState) const
{
UInt32 price;
if(repIndex == 0)
{
price = _isRepG0[state.Index].GetPrice0();
price += _isRep0Long[state.Index][posState].GetPrice1();
}
else
{
price = _isRepG0[state.Index].GetPrice1();
if (repIndex == 1)
price += _isRepG1[state.Index].GetPrice0();
else
{
price += _isRepG1[state.Index].GetPrice1();
price += _isRepG2[state.Index].GetPrice(repIndex - 2);
}
}
return price;
}
UInt32 GetRepPrice(UInt32 repIndex, UInt32 len, CState state, UInt32 posState) const
{
return _repMatchLenEncoder.GetPrice(len - kMatchMinLen, posState) +
GetPureRepPrice(repIndex, state, posState);
}
/*
UInt32 GetPosLen2Price(UInt32 pos, UInt32 posState) const
{
if (pos >= kNumFullDistances)
return kIfinityPrice;
return _distancesPrices[0][pos] + _lenEncoder.GetPrice(0, posState);
}
UInt32 GetPosLen3Price(UInt32 pos, UInt32 len, UInt32 posState) const
{
UInt32 price;
UInt32 lenToPosState = GetLenToPosState(len);
if (pos < kNumFullDistances)
price = _distancesPrices[lenToPosState][pos];
else
price = _posSlotPrices[lenToPosState][GetPosSlot2(pos)] +
_alignPrices[pos & kAlignMask];
return price + _lenEncoder.GetPrice(len - kMatchMinLen, posState);
}
*/
UInt32 GetPosLenPrice(UInt32 pos, UInt32 len, UInt32 posState) const
{
UInt32 price;
UInt32 lenToPosState = GetLenToPosState(len);
if (pos < kNumFullDistances)
price = _distancesPrices[lenToPosState][pos];
else
price = _posSlotPrices[lenToPosState][GetPosSlot2(pos)] +
_alignPrices[pos & kAlignMask];
return price + _lenEncoder.GetPrice(len - kMatchMinLen, posState);
}
UInt32 Backward(UInt32 &backRes, UInt32 cur);
HRESULT GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes);
HRESULT GetOptimumFast(UInt32 position, UInt32 &backRes, UInt32 &lenRes);
void FillDistancesPrices();
void FillAlignPrices();
void ReleaseMFStream()
{
if (_matchFinder && _needReleaseMFStream)
{
_matchFinder->ReleaseStream();
_needReleaseMFStream = false;
}
}
void ReleaseStreams()
{
ReleaseMFStream();
ReleaseOutStream();
}
HRESULT Flush(UInt32 nowPos);
class CCoderReleaser
{
CEncoder *_coder;
public:
CCoderReleaser(CEncoder *coder): _coder(coder) {}
~CCoderReleaser()
{
_coder->ReleaseStreams();
}
};
friend class CCoderReleaser;
void WriteEndMarker(UInt32 posState);
public:
CEncoder();
void SetWriteEndMarkerMode(bool writeEndMarker)
{ _writeEndMark= writeEndMarker; }
HRESULT Create();
MY_UNKNOWN_IMP3(
ICompressSetOutStream,
ICompressSetCoderProperties,
ICompressWriteCoderProperties
)
HRESULT Init();
// ICompressCoder interface
HRESULT SetStreams(ISequentialInStream *inStream,
ISequentialOutStream *outStream,
const UInt64 *inSize, const UInt64 *outSize);
HRESULT CodeOneBlock(UInt64 *inSize, UInt64 *outSize, Int32 *finished);
HRESULT CodeReal(ISequentialInStream *inStream,
ISequentialOutStream *outStream,
const UInt64 *inSize, const UInt64 *outSize,
ICompressProgressInfo *progress);
// ICompressCoder interface
STDMETHOD(Code)(ISequentialInStream *inStream,
ISequentialOutStream *outStream,
const UInt64 *inSize, const UInt64 *outSize,
ICompressProgressInfo *progress);
// ICompressSetCoderProperties2
STDMETHOD(SetCoderProperties)(const PROPID *propIDs,
const PROPVARIANT *properties, UInt32 numProperties);
// ICompressWriteCoderProperties
STDMETHOD(WriteCoderProperties)(ISequentialOutStream *outStream);
STDMETHOD(SetOutStream)(ISequentialOutStream *outStream);
STDMETHOD(ReleaseOutStream)();
virtual ~CEncoder() {}
};
}}
#endif

View File

@@ -0,0 +1,8 @@
// StdAfx.h
#ifndef __STDAFX_H
#define __STDAFX_H
#include "../../../Common/MyWindows.h"
#endif

View File

@@ -0,0 +1,205 @@
// Compress/RangeCoder/RangeCoder.h
#ifndef __COMPRESS_RANGECODER_H
#define __COMPRESS_RANGECODER_H
#include "../../Common/InBuffer.h"
#include "../../Common/OutBuffer.h"
namespace NCompress {
namespace NRangeCoder {
const int kNumTopBits = 24;
const UInt32 kTopValue = (1 << kNumTopBits);
class CEncoder
{
UInt32 _cacheSize;
Byte _cache;
public:
UInt64 Low;
UInt32 Range;
COutBuffer Stream;
bool Create(UInt32 bufferSize) { return Stream.Create(bufferSize); }
void SetStream(ISequentialOutStream *stream) { Stream.SetStream(stream); }
void Init()
{
Stream.Init();
Low = 0;
Range = 0xFFFFFFFF;
_cacheSize = 1;
_cache = 0;
}
void FlushData()
{
// Low += 1;
for(int i = 0; i < 5; i++)
ShiftLow();
}
HRESULT FlushStream() { return Stream.Flush(); }
void ReleaseStream() { Stream.ReleaseStream(); }
void Encode(UInt32 start, UInt32 size, UInt32 total)
{
Low += start * (Range /= total);
Range *= size;
while (Range < kTopValue)
{
Range <<= 8;
ShiftLow();
}
}
void ShiftLow()
{
if ((UInt32)Low < (UInt32)0xFF000000 || (int)(Low >> 32) != 0)
{
Byte temp = _cache;
do
{
Stream.WriteByte((Byte)(temp + (Byte)(Low >> 32)));
temp = 0xFF;
}
while(--_cacheSize != 0);
_cache = (Byte)((UInt32)Low >> 24);
}
_cacheSize++;
Low = (UInt32)Low << 8;
}
void EncodeDirectBits(UInt32 value, int numTotalBits)
{
for (int i = numTotalBits - 1; i >= 0; i--)
{
Range >>= 1;
if (((value >> i) & 1) == 1)
Low += Range;
if (Range < kTopValue)
{
Range <<= 8;
ShiftLow();
}
}
}
void EncodeBit(UInt32 size0, UInt32 numTotalBits, UInt32 symbol)
{
UInt32 newBound = (Range >> numTotalBits) * size0;
if (symbol == 0)
Range = newBound;
else
{
Low += newBound;
Range -= newBound;
}
while (Range < kTopValue)
{
Range <<= 8;
ShiftLow();
}
}
UInt64 GetProcessedSize() { return Stream.GetProcessedSize() + _cacheSize + 4; }
};
class CDecoder
{
public:
CInBuffer Stream;
UInt32 Range;
UInt32 Code;
bool Create(UInt32 bufferSize) { return Stream.Create(bufferSize); }
void Normalize()
{
while (Range < kTopValue)
{
Code = (Code << 8) | Stream.ReadByte();
Range <<= 8;
}
}
void SetStream(ISequentialInStream *stream) { Stream.SetStream(stream); }
void Init()
{
Stream.Init();
Code = 0;
Range = 0xFFFFFFFF;
for(int i = 0; i < 5; i++)
Code = (Code << 8) | Stream.ReadByte();
}
void ReleaseStream() { Stream.ReleaseStream(); }
UInt32 GetThreshold(UInt32 total)
{
return (Code) / ( Range /= total);
}
void Decode(UInt32 start, UInt32 size)
{
Code -= start * Range;
Range *= size;
Normalize();
}
UInt32 DecodeDirectBits(int numTotalBits)
{
UInt32 range = Range;
UInt32 code = Code;
UInt32 result = 0;
for (int i = numTotalBits; i != 0; i--)
{
range >>= 1;
/*
result <<= 1;
if (code >= range)
{
code -= range;
result |= 1;
}
*/
UInt32 t = (code - range) >> 31;
code -= range & (t - 1);
result = (result << 1) | (1 - t);
if (range < kTopValue)
{
code = (code << 8) | Stream.ReadByte();
range <<= 8;
}
}
Range = range;
Code = code;
return result;
}
UInt32 DecodeBit(UInt32 size0, UInt32 numTotalBits)
{
UInt32 newBound = (Range >> numTotalBits) * size0;
UInt32 symbol;
if (Code < newBound)
{
symbol = 0;
Range = newBound;
}
else
{
symbol = 1;
Code -= newBound;
Range -= newBound;
}
Normalize();
return symbol;
}
UInt64 GetProcessedSize() {return Stream.GetProcessedSize(); }
};
}}
#endif

View File

@@ -0,0 +1,80 @@
// Compress/RangeCoder/RangeCoderBit.cpp
#include "StdAfx.h"
#include "RangeCoderBit.h"
namespace NCompress {
namespace NRangeCoder {
UInt32 CPriceTables::ProbPrices[kBitModelTotal >> kNumMoveReducingBits];
static CPriceTables g_PriceTables;
CPriceTables::CPriceTables() { Init(); }
void CPriceTables::Init()
{
const int kNumBits = (kNumBitModelTotalBits - kNumMoveReducingBits);
for(int i = kNumBits - 1; i >= 0; i--)
{
UInt32 start = 1 << (kNumBits - i - 1);
UInt32 end = 1 << (kNumBits - i);
for (UInt32 j = start; j < end; j++)
ProbPrices[j] = (i << kNumBitPriceShiftBits) +
(((end - j) << kNumBitPriceShiftBits) >> (kNumBits - i - 1));
}
/*
// simplest: bad solution
for(UInt32 i = 1; i < (kBitModelTotal >> kNumMoveReducingBits) - 1; i++)
ProbPrices[i] = kBitPrice;
*/
/*
const double kDummyMultMid = (1.0 / kBitPrice) / 2;
const double kDummyMultMid = 0;
// float solution
double ln2 = log(double(2));
double lnAll = log(double(kBitModelTotal >> kNumMoveReducingBits));
for(UInt32 i = 1; i < (kBitModelTotal >> kNumMoveReducingBits) - 1; i++)
ProbPrices[i] = UInt32((fabs(lnAll - log(double(i))) / ln2 + kDummyMultMid) * kBitPrice);
*/
/*
// experimental, slow, solution:
for(UInt32 i = 1; i < (kBitModelTotal >> kNumMoveReducingBits) - 1; i++)
{
const int kCyclesBits = 5;
const UInt32 kCycles = (1 << kCyclesBits);
UInt32 range = UInt32(-1);
UInt32 bitCount = 0;
for (UInt32 j = 0; j < kCycles; j++)
{
range >>= (kNumBitModelTotalBits - kNumMoveReducingBits);
range *= i;
while(range < (1 << 31))
{
range <<= 1;
bitCount++;
}
}
bitCount <<= kNumBitPriceShiftBits;
range -= (1 << 31);
for (int k = kNumBitPriceShiftBits - 1; k >= 0; k--)
{
range <<= 1;
if (range > (1 << 31))
{
bitCount += (1 << k);
range -= (1 << 31);
}
}
ProbPrices[i] = (bitCount
// + (1 << (kCyclesBits - 1))
) >> kCyclesBits;
}
*/
}
}}

View File

@@ -0,0 +1,120 @@
// Compress/RangeCoder/RangeCoderBit.h
#ifndef __COMPRESS_RANGECODER_BIT_H
#define __COMPRESS_RANGECODER_BIT_H
#include "RangeCoder.h"
namespace NCompress {
namespace NRangeCoder {
const int kNumBitModelTotalBits = 11;
const UInt32 kBitModelTotal = (1 << kNumBitModelTotalBits);
const int kNumMoveReducingBits = 2;
const int kNumBitPriceShiftBits = 6;
const UInt32 kBitPrice = 1 << kNumBitPriceShiftBits;
class CPriceTables
{
public:
static UInt32 ProbPrices[kBitModelTotal >> kNumMoveReducingBits];
static void Init();
CPriceTables();
};
template <int numMoveBits>
class CBitModel
{
public:
UInt32 Prob;
void UpdateModel(UInt32 symbol)
{
/*
Prob -= (Prob + ((symbol - 1) & ((1 << numMoveBits) - 1))) >> numMoveBits;
Prob += (1 - symbol) << (kNumBitModelTotalBits - numMoveBits);
*/
if (symbol == 0)
Prob += (kBitModelTotal - Prob) >> numMoveBits;
else
Prob -= (Prob) >> numMoveBits;
}
public:
void Init() { Prob = kBitModelTotal / 2; }
};
template <int numMoveBits>
class CBitEncoder: public CBitModel<numMoveBits>
{
public:
void Encode(CEncoder *encoder, UInt32 symbol)
{
/*
encoder->EncodeBit(this->Prob, kNumBitModelTotalBits, symbol);
this->UpdateModel(symbol);
*/
UInt32 newBound = (encoder->Range >> kNumBitModelTotalBits) * this->Prob;
if (symbol == 0)
{
encoder->Range = newBound;
this->Prob += (kBitModelTotal - this->Prob) >> numMoveBits;
}
else
{
encoder->Low += newBound;
encoder->Range -= newBound;
this->Prob -= (this->Prob) >> numMoveBits;
}
if (encoder->Range < kTopValue)
{
encoder->Range <<= 8;
encoder->ShiftLow();
}
}
UInt32 GetPrice(UInt32 symbol) const
{
return CPriceTables::ProbPrices[
(((this->Prob - symbol) ^ ((-(int)symbol))) & (kBitModelTotal - 1)) >> kNumMoveReducingBits];
}
UInt32 GetPrice0() const { return CPriceTables::ProbPrices[this->Prob >> kNumMoveReducingBits]; }
UInt32 GetPrice1() const { return CPriceTables::ProbPrices[(kBitModelTotal - this->Prob) >> kNumMoveReducingBits]; }
};
template <int numMoveBits>
class CBitDecoder: public CBitModel<numMoveBits>
{
public:
UInt32 Decode(CDecoder *decoder)
{
UInt32 newBound = (decoder->Range >> kNumBitModelTotalBits) * this->Prob;
if (decoder->Code < newBound)
{
decoder->Range = newBound;
this->Prob += (kBitModelTotal - this->Prob) >> numMoveBits;
if (decoder->Range < kTopValue)
{
decoder->Code = (decoder->Code << 8) | decoder->Stream.ReadByte();
decoder->Range <<= 8;
}
return 0;
}
else
{
decoder->Range -= newBound;
decoder->Code -= newBound;
this->Prob -= (this->Prob) >> numMoveBits;
if (decoder->Range < kTopValue)
{
decoder->Code = (decoder->Code << 8) | decoder->Stream.ReadByte();
decoder->Range <<= 8;
}
return 1;
}
}
};
}}
#endif

View File

@@ -0,0 +1,161 @@
// Compress/RangeCoder/RangeCoderBitTree.h
#ifndef __COMPRESS_RANGECODER_BIT_TREE_H
#define __COMPRESS_RANGECODER_BIT_TREE_H
#include "RangeCoderBit.h"
#include "RangeCoderOpt.h"
namespace NCompress {
namespace NRangeCoder {
template <int numMoveBits, int NumBitLevels>
class CBitTreeEncoder
{
CBitEncoder<numMoveBits> Models[1 << NumBitLevels];
public:
void Init()
{
for(UInt32 i = 1; i < (1 << NumBitLevels); i++)
Models[i].Init();
}
void Encode(CEncoder *rangeEncoder, UInt32 symbol)
{
UInt32 modelIndex = 1;
for (int bitIndex = NumBitLevels; bitIndex != 0 ;)
{
bitIndex--;
UInt32 bit = (symbol >> bitIndex) & 1;
Models[modelIndex].Encode(rangeEncoder, bit);
modelIndex = (modelIndex << 1) | bit;
}
};
void ReverseEncode(CEncoder *rangeEncoder, UInt32 symbol)
{
UInt32 modelIndex = 1;
for (int i = 0; i < NumBitLevels; i++)
{
UInt32 bit = symbol & 1;
Models[modelIndex].Encode(rangeEncoder, bit);
modelIndex = (modelIndex << 1) | bit;
symbol >>= 1;
}
}
UInt32 GetPrice(UInt32 symbol) const
{
symbol |= (1 << NumBitLevels);
UInt32 price = 0;
while (symbol != 1)
{
price += Models[symbol >> 1].GetPrice(symbol & 1);
symbol >>= 1;
}
return price;
}
UInt32 ReverseGetPrice(UInt32 symbol) const
{
UInt32 price = 0;
UInt32 modelIndex = 1;
for (int i = NumBitLevels; i != 0; i--)
{
UInt32 bit = symbol & 1;
symbol >>= 1;
price += Models[modelIndex].GetPrice(bit);
modelIndex = (modelIndex << 1) | bit;
}
return price;
}
};
template <int numMoveBits, int NumBitLevels>
class CBitTreeDecoder
{
CBitDecoder<numMoveBits> Models[1 << NumBitLevels];
public:
void Init()
{
for(UInt32 i = 1; i < (1 << NumBitLevels); i++)
Models[i].Init();
}
UInt32 Decode(CDecoder *rangeDecoder)
{
UInt32 modelIndex = 1;
RC_INIT_VAR
for(int bitIndex = NumBitLevels; bitIndex != 0; bitIndex--)
{
// modelIndex = (modelIndex << 1) + Models[modelIndex].Decode(rangeDecoder);
RC_GETBIT(numMoveBits, Models[modelIndex].Prob, modelIndex)
}
RC_FLUSH_VAR
return modelIndex - (1 << NumBitLevels);
};
UInt32 ReverseDecode(CDecoder *rangeDecoder)
{
UInt32 modelIndex = 1;
UInt32 symbol = 0;
RC_INIT_VAR
for(int bitIndex = 0; bitIndex < NumBitLevels; bitIndex++)
{
// UInt32 bit = Models[modelIndex].Decode(rangeDecoder);
// modelIndex <<= 1;
// modelIndex += bit;
// symbol |= (bit << bitIndex);
RC_GETBIT2(numMoveBits, Models[modelIndex].Prob, modelIndex, ; , symbol |= (1 << bitIndex))
}
RC_FLUSH_VAR
return symbol;
}
};
template <int numMoveBits>
void ReverseBitTreeEncode(CBitEncoder<numMoveBits> *Models,
CEncoder *rangeEncoder, int NumBitLevels, UInt32 symbol)
{
UInt32 modelIndex = 1;
for (int i = 0; i < NumBitLevels; i++)
{
UInt32 bit = symbol & 1;
Models[modelIndex].Encode(rangeEncoder, bit);
modelIndex = (modelIndex << 1) | bit;
symbol >>= 1;
}
}
template <int numMoveBits>
UInt32 ReverseBitTreeGetPrice(CBitEncoder<numMoveBits> *Models,
UInt32 NumBitLevels, UInt32 symbol)
{
UInt32 price = 0;
UInt32 modelIndex = 1;
for (int i = NumBitLevels; i != 0; i--)
{
UInt32 bit = symbol & 1;
symbol >>= 1;
price += Models[modelIndex].GetPrice(bit);
modelIndex = (modelIndex << 1) | bit;
}
return price;
}
template <int numMoveBits>
UInt32 ReverseBitTreeDecode(CBitDecoder<numMoveBits> *Models,
CDecoder *rangeDecoder, int NumBitLevels)
{
UInt32 modelIndex = 1;
UInt32 symbol = 0;
RC_INIT_VAR
for(int bitIndex = 0; bitIndex < NumBitLevels; bitIndex++)
{
// UInt32 bit = Models[modelIndex].Decode(rangeDecoder);
// modelIndex <<= 1;
// modelIndex += bit;
// symbol |= (bit << bitIndex);
RC_GETBIT2(numMoveBits, Models[modelIndex].Prob, modelIndex, ; , symbol |= (1 << bitIndex))
}
RC_FLUSH_VAR
return symbol;
}
}}
#endif

View File

@@ -0,0 +1,31 @@
// Compress/RangeCoder/RangeCoderOpt.h
#ifndef __COMPRESS_RANGECODER_OPT_H
#define __COMPRESS_RANGECODER_OPT_H
#define RC_INIT_VAR \
UInt32 range = rangeDecoder->Range; \
UInt32 code = rangeDecoder->Code;
#define RC_FLUSH_VAR \
rangeDecoder->Range = range; \
rangeDecoder->Code = code;
#define RC_NORMALIZE \
if (range < NCompress::NRangeCoder::kTopValue) \
{ code = (code << 8) | rangeDecoder->Stream.ReadByte(); range <<= 8; }
#define RC_GETBIT2(numMoveBits, prob, mi, A0, A1) \
{ UInt32 bound = (range >> NCompress::NRangeCoder::kNumBitModelTotalBits) * prob; \
if (code < bound) \
{ A0; range = bound; \
prob += (NCompress::NRangeCoder::kBitModelTotal - prob) >> numMoveBits; \
mi <<= 1; } \
else \
{ A1; range -= bound; code -= bound; prob -= (prob) >> numMoveBits; \
mi = (mi + mi) + 1; }} \
RC_NORMALIZE
#define RC_GETBIT(numMoveBits, prob, mi) RC_GETBIT2(numMoveBits, prob, mi, ; , ;)
#endif

View File

@@ -0,0 +1,6 @@
// StdAfx.h
#ifndef __STDAFX_H
#define __STDAFX_H
#endif