From 0f57aea2df2a1272b57de896e2f6aad80809e5d3 Mon Sep 17 00:00:00 2001 From: Bastien Bouclet Date: Mon, 8 Apr 2019 19:24:00 +0200 Subject: [PATCH] COMMON: Use a prefix table to speed up the Huffman decoder Symbols for codes shorter than the prefix table index width are stored in the table. All the entries in the table with an index starting with the code are set to the symbol value. That way, when decoding it is possible to get the number of bits corresponding to the table width from the bitstream and directly find the symbol value. Longer code still need to be searched for in the codes list. --- common/huffman.cpp | 71 ------------------------- common/huffman.h | 118 ++++++++++++++++++++++++++++++++--------- common/module.mk | 1 - image/codecs/svq1.cpp | 12 ++--- image/codecs/svq1.h | 15 +++--- test/common/huffman.h | 51 +----------------- video/bink_decoder.cpp | 2 +- video/bink_decoder.h | 3 +- video/psx_decoder.cpp | 8 +-- video/psx_decoder.h | 7 ++- 10 files changed, 123 insertions(+), 165 deletions(-) delete mode 100644 common/huffman.cpp diff --git a/common/huffman.cpp b/common/huffman.cpp deleted file mode 100644 index 3268ddf2513..00000000000 --- a/common/huffman.cpp +++ /dev/null @@ -1,71 +0,0 @@ -/* ScummVM - Graphic Adventure Engine - * - * ScummVM is the legal property of its developers, whose names - * are too numerous to list here. Please refer to the COPYRIGHT - * file distributed with this source distribution. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * - */ - -// Based on eos' Huffman code - -#include "common/huffman.h" -#include "common/util.h" -#include "common/textconsole.h" -#include "common/bitstream.h" - -namespace Common { - -Huffman::Symbol::Symbol(uint32 c, uint32 s) : code(c), symbol(s) { -} - - -Huffman::Huffman(uint8 maxLength, uint32 codeCount, const uint32 *codes, const uint8 *lengths, const uint32 *symbols) { - assert(codeCount > 0); - - assert(codes); - assert(lengths); - - if (maxLength == 0) - for (uint32 i = 0; i < codeCount; i++) - maxLength = MAX(maxLength, lengths[i]); - - assert(maxLength <= 32); - - _codes.resize(maxLength); - _symbols.resize(codeCount); - - for (uint32 i = 0; i < codeCount; i++) { - // The symbol. If none were specified, just assume it's identical to the code index - uint32 symbol = symbols ? symbols[i] : i; - - // Put the code and symbol into the correct list - _codes[lengths[i] - 1].push_back(Symbol(codes[i], symbol)); - - // And put the pointer to the symbol/code struct into the symbol list. - _symbols[i] = &_codes[lengths[i] - 1].back(); - } -} - -Huffman::~Huffman() { -} - -void Huffman::setSymbols(const uint32 *symbols) { - for (uint32 i = 0; i < _symbols.size(); i++) - _symbols[i]->symbol = symbols ? *symbols++ : i; -} - -} // End of namespace Common diff --git a/common/huffman.h b/common/huffman.h index b4de5ab4830..052c7bf8251 100644 --- a/common/huffman.h +++ b/common/huffman.h @@ -20,7 +20,7 @@ * */ -// Based on eos' Huffman code +// Based on xoreos' Huffman code #ifndef COMMON_HUFFMAN_H #define COMMON_HUFFMAN_H @@ -31,12 +31,22 @@ namespace Common { +inline uint32 REVERSEBITS(uint32 x) { + x = (((x & ~0x55555555) >> 1) | ((x & 0x55555555) << 1)); + x = (((x & ~0x33333333) >> 2) | ((x & 0x33333333) << 2)); + x = (((x & ~0x0F0F0F0F) >> 4) | ((x & 0x0F0F0F0F) << 4)); + x = (((x & ~0x00FF00FF) >> 8) | ((x & 0x00FF00FF) << 8)); + + return((x >> 16) | (x << 16)); +} + /** * Huffman bitstream decoding * * Used in engines: * - scumm */ +template class Huffman { public: /** Construct a Huffman decoder. @@ -48,47 +58,107 @@ public: * @param symbols The symbols. If 0, assume they are identical to the code indices. */ Huffman(uint8 maxLength, uint32 codeCount, const uint32 *codes, const uint8 *lengths, const uint32 *symbols = nullptr); - ~Huffman(); - - /** Modify the codes' symbols. */ - void setSymbols(const uint32 *symbols = nullptr); /** Return the next symbol in the bitstream. */ - template - uint32 getSymbol(BITSTREAM &bits) const { - uint32 code = 0; - - for (uint32 i = 0; i < _codes.size(); i++) { - bits.addBit(code, i); - - for (CodeList::const_iterator cCode = _codes[i].begin(); cCode != _codes[i].end(); ++cCode) - if (code == cCode->code) - return cCode->symbol; - } - - error("Unknown Huffman code"); - return 0; - } + uint32 getSymbol(BITSTREAM &bits) const; private: struct Symbol { uint32 code; uint32 symbol; - Symbol(uint32 c, uint32 s); + Symbol(uint32 c, uint32 s) : code(c), symbol(s) {} }; typedef List CodeList; typedef Array CodeLists; - typedef Array SymbolList; /** Lists of codes and their symbols, sorted by code length. */ CodeLists _codes; - /** Sorted list of pointers to the symbols. */ - SymbolList _symbols; + /** Prefix lookup table used to speed up the decoding of short codes. */ + struct PrefixEntry { + uint32 symbol; + uint8 length; + + PrefixEntry() : length(0xFF) {} + }; + + static const uint8 _prefixTableBits = 8; + PrefixEntry _prefixTable[1 << _prefixTableBits]; }; +template +Huffman::Huffman(uint8 maxLength, uint32 codeCount, const uint32 *codes, const uint8 *lengths, const uint32 *symbols) { + assert(codeCount > 0); + + assert(codes); + assert(lengths); + + if (maxLength == 0) + for (uint32 i = 0; i < codeCount; i++) + maxLength = MAX(maxLength, lengths[i]); + + assert(maxLength <= 32); + + // Codes that don't fit in the prefix table are stored in the _codes array + _codes.resize(MAX(maxLength - _prefixTableBits, 0)); + + for (uint i = 0; i < codeCount; i++) { + uint8 length = lengths[i]; + + // The symbol. If none were specified, just assume it's identical to the code index + uint32 symbol = symbols ? symbols[i] : i; + + if (length <= _prefixTableBits) { + // Short codes go in the prefix lookup table. Set all the entries in the table + // with an index starting with the code to the symbol value. + uint32 startIndex; + if (BITSTREAM::isMSB2LSB()) { + startIndex = codes[i] << (_prefixTableBits - length); + } else { + startIndex = REVERSEBITS(codes[i]) >> (32 - _prefixTableBits); + } + + uint32 endIndex = startIndex | ((1 << (_prefixTableBits - length)) - 1); + + for (uint32 j = startIndex; j <= endIndex; j++) { + uint32 index = BITSTREAM::isMSB2LSB() ? j : REVERSEBITS(j) >> (32 - _prefixTableBits); + _prefixTable[index].symbol = symbol; + _prefixTable[index].length = length; + } + } else { + // Put the code and symbol into the correct list for the length + _codes[lengths[i] - 1 - _prefixTableBits].push_back(Symbol(codes[i], symbol)); + } + } +} + +template +uint32 Huffman::getSymbol(BITSTREAM &bits) const { + uint32 code = bits.peekBits(_prefixTableBits); + + uint8 length = _prefixTable[code].length; + + if (length != 0xFF) { + bits.skip(length); + return _prefixTable[code].symbol; + } else { + bits.skip(_prefixTableBits); + + for (uint32 i = 0; i < _codes.size(); i++) { + bits.addBit(code, i + _prefixTableBits); + + for (typename CodeList::const_iterator cCode = _codes[i].begin(); cCode != _codes[i].end(); ++cCode) + if (code == cCode->code) + return cCode->symbol; + } + } + + error("Unknown Huffman code"); + return 0; +} + } // End of namespace Common #endif // COMMON_HUFFMAN_H diff --git a/common/module.mk b/common/module.mk index 6742fef3e8c..f456604ebb7 100644 --- a/common/module.mk +++ b/common/module.mk @@ -49,7 +49,6 @@ MODULE_OBJS += \ cosinetables.o \ dct.o \ fft.o \ - huffman.o \ rdft.o \ sinetables.o diff --git a/image/codecs/svq1.cpp b/image/codecs/svq1.cpp index 2d860708016..eba5fb872aa 100644 --- a/image/codecs/svq1.cpp +++ b/image/codecs/svq1.cpp @@ -56,16 +56,16 @@ SVQ1Decoder::SVQ1Decoder(uint16 width, uint16 height) { _last[2] = 0; // Setup Variable Length Code Tables - _blockType = new Common::Huffman(0, 4, s_svq1BlockTypeCodes, s_svq1BlockTypeLengths); + _blockType = new HuffmanDecoder(0, 4, s_svq1BlockTypeCodes, s_svq1BlockTypeLengths); for (int i = 0; i < 6; i++) { - _intraMultistage[i] = new Common::Huffman(0, 8, s_svq1IntraMultistageCodes[i], s_svq1IntraMultistageLengths[i]); - _interMultistage[i] = new Common::Huffman(0, 8, s_svq1InterMultistageCodes[i], s_svq1InterMultistageLengths[i]); + _intraMultistage[i] = new HuffmanDecoder(0, 8, s_svq1IntraMultistageCodes[i], s_svq1IntraMultistageLengths[i]); + _interMultistage[i] = new HuffmanDecoder(0, 8, s_svq1InterMultistageCodes[i], s_svq1InterMultistageLengths[i]); } - _intraMean = new Common::Huffman(0, 256, s_svq1IntraMeanCodes, s_svq1IntraMeanLengths); - _interMean = new Common::Huffman(0, 512, s_svq1InterMeanCodes, s_svq1InterMeanLengths); - _motionComponent = new Common::Huffman(0, 33, s_svq1MotionComponentCodes, s_svq1MotionComponentLengths); + _intraMean = new HuffmanDecoder(0, 256, s_svq1IntraMeanCodes, s_svq1IntraMeanLengths); + _interMean = new HuffmanDecoder(0, 512, s_svq1InterMeanCodes, s_svq1InterMeanLengths); + _motionComponent = new HuffmanDecoder(0, 33, s_svq1MotionComponentCodes, s_svq1MotionComponentLengths); } SVQ1Decoder::~SVQ1Decoder() { diff --git a/image/codecs/svq1.h b/image/codecs/svq1.h index 148501d17f2..4a005761a52 100644 --- a/image/codecs/svq1.h +++ b/image/codecs/svq1.h @@ -27,6 +27,7 @@ #include "image/codecs/codec.h" namespace Common { +template class Huffman; struct Point; } @@ -53,12 +54,14 @@ private: byte *_last[3]; - Common::Huffman *_blockType; - Common::Huffman *_intraMultistage[6]; - Common::Huffman *_interMultistage[6]; - Common::Huffman *_intraMean; - Common::Huffman *_interMean; - Common::Huffman *_motionComponent; + typedef Common::Huffman HuffmanDecoder; + + HuffmanDecoder *_blockType; + HuffmanDecoder *_intraMultistage[6]; + HuffmanDecoder *_interMultistage[6]; + HuffmanDecoder *_intraMean; + HuffmanDecoder *_interMean; + HuffmanDecoder *_motionComponent; bool svq1DecodeBlockIntra(Common::BitStream32BEMSB *s, byte *pixels, int pitch); bool svq1DecodeBlockNonIntra(Common::BitStream32BEMSB *s, byte *pixels, int pitch); diff --git a/test/common/huffman.h b/test/common/huffman.h index 53353aaa60e..3e3956a3213 100644 --- a/test/common/huffman.h +++ b/test/common/huffman.h @@ -32,7 +32,7 @@ class HuffmanTestSuite : public CxxTest::TestSuite { const uint32 codes[] = {0x2, 0x3, 0x3, 0x0, 0x2}; const uint32 symbols[] = {0xA, 0xB, 0xC, 0xD, 0xE}; - Common::Huffman h(maxLength, codeCount, codes, lengths, symbols); + Common::Huffman h(maxLength, codeCount, codes, lengths, symbols); byte input[] = {0x4F, 0x20}; // Provided input... @@ -78,7 +78,7 @@ class HuffmanTestSuite : public CxxTest::TestSuite { const uint8 lengths[] = {3,3,2,2,2}; const uint32 codes[] = {0x2, 0x3, 0x3, 0x0, 0x2}; - Common::Huffman h(0, codeCount, codes, lengths, 0); + Common::Huffman h(0, codeCount, codes, lengths, 0); byte input[] = {0x4F, 0x20}; uint32 expected[] = {0, 1, 2, 3, 4, 3 ,3}; @@ -94,51 +94,4 @@ class HuffmanTestSuite : public CxxTest::TestSuite { TS_ASSERT_EQUALS(h.getSymbol(bs), expected[5]); TS_ASSERT_EQUALS(h.getSymbol(bs), expected[6]); } - - void test_get_after_set_symbols() { - - /* - * Another variation of test_get_with_full_symbols. - * I use the setSymbols method to define, a posteriori, - * an alphabet to be used in place of array indices. - * The encoding is, at first, - * 0=010 - * 1=011 - * 2=11 - * 3=00 - * 4=10 - * (=array indices). - */ - - uint32 codeCount = 5; - const uint8 lengths[] = {3,3,2,2,2}; - const uint32 codes[] = {0x2, 0x3, 0x3, 0x0, 0x2}; - - Common::Huffman h(0, codeCount, codes, lengths, 0); - - const uint32 symbols[] = {0xA, 0xB, 0xC, 0xD, 0xE}; - h.setSymbols(symbols); - - byte input[] = {0x4F, 0x20}; - uint32 expected[] = {0xA, 0xB, 0xC, 0xD, 0xE, 0xD, 0xD}; - - Common::MemoryReadStream ms(input, sizeof(input)); - Common::BitStream8MSB bs(ms); - - /* New symbols: - * A=010 - * B=011 - * C=11 - * D=00 - * E=10 - */ - - TS_ASSERT_EQUALS(h.getSymbol(bs), expected[0]); - TS_ASSERT_EQUALS(h.getSymbol(bs), expected[1]); - TS_ASSERT_EQUALS(h.getSymbol(bs), expected[2]); - TS_ASSERT_EQUALS(h.getSymbol(bs), expected[3]); - TS_ASSERT_EQUALS(h.getSymbol(bs), expected[4]); - TS_ASSERT_EQUALS(h.getSymbol(bs), expected[5]); - TS_ASSERT_EQUALS(h.getSymbol(bs), expected[6]); - } }; diff --git a/video/bink_decoder.cpp b/video/bink_decoder.cpp index 406c6e4d34e..4074043768a 100644 --- a/video/bink_decoder.cpp +++ b/video/bink_decoder.cpp @@ -594,7 +594,7 @@ void BinkDecoder::BinkVideoTrack::deinitBundles() { void BinkDecoder::BinkVideoTrack::initHuffman() { for (int i = 0; i < 16; i++) - _huffman[i] = new Common::Huffman(binkHuffmanLengths[i][15], 16, binkHuffmanCodes[i], binkHuffmanLengths[i]); + _huffman[i] = new Common::Huffman(binkHuffmanLengths[i][15], 16, binkHuffmanCodes[i], binkHuffmanLengths[i]); } byte BinkDecoder::BinkVideoTrack::getHuffmanSymbol(VideoFrame &video, Huffman &huffman) { diff --git a/video/bink_decoder.h b/video/bink_decoder.h index 29d16020b1c..11694314b73 100644 --- a/video/bink_decoder.h +++ b/video/bink_decoder.h @@ -46,6 +46,7 @@ class QueuingAudioStream; namespace Common { class SeekableReadStream; +template class Huffman; class RDFT; @@ -247,7 +248,7 @@ private: Bundle _bundles[kSourceMAX]; ///< Bundles for decoding all data types. - Common::Huffman *_huffman[16]; ///< The 16 Huffman codebooks used in Bink decoding. + Common::Huffman *_huffman[16]; ///< The 16 Huffman codebooks used in Bink decoding. /** Huffman codebooks to use for decoding high nibbles in color data types. */ Huffman _colHighHuffman[16]; diff --git a/video/psx_decoder.cpp b/video/psx_decoder.cpp index 562da885b1e..ef42a728733 100644 --- a/video/psx_decoder.cpp +++ b/video/psx_decoder.cpp @@ -438,9 +438,9 @@ PSXStreamDecoder::PSXVideoTrack::PSXVideoTrack(Common::SeekableReadStream *first _endOfTrack = false; _curFrame = -1; - _acHuffman = new Common::Huffman(0, AC_CODE_COUNT, s_huffmanACCodes, s_huffmanACLengths, s_huffmanACSymbols); - _dcHuffmanChroma = new Common::Huffman(0, DC_CODE_COUNT, s_huffmanDCChromaCodes, s_huffmanDCChromaLengths, s_huffmanDCSymbols); - _dcHuffmanLuma = new Common::Huffman(0, DC_CODE_COUNT, s_huffmanDCLumaCodes, s_huffmanDCLumaLengths, s_huffmanDCSymbols); + _acHuffman = new HuffmanDecoder(0, AC_CODE_COUNT, s_huffmanACCodes, s_huffmanACLengths, s_huffmanACSymbols); + _dcHuffmanChroma = new HuffmanDecoder(0, DC_CODE_COUNT, s_huffmanDCChromaCodes, s_huffmanDCChromaLengths, s_huffmanDCSymbols); + _dcHuffmanLuma = new HuffmanDecoder(0, DC_CODE_COUNT, s_huffmanDCLumaCodes, s_huffmanDCLumaLengths, s_huffmanDCSymbols); } PSXStreamDecoder::PSXVideoTrack::~PSXVideoTrack() { @@ -552,7 +552,7 @@ int PSXStreamDecoder::PSXVideoTrack::readDC(Common::BitStreamMemory16LEMSB *bits // Version 3 has it stored as huffman codes as a difference from the previous DC value - Common::Huffman *huffman = (plane == kPlaneY) ? _dcHuffmanLuma : _dcHuffmanChroma; + HuffmanDecoder *huffman = (plane == kPlaneY) ? _dcHuffmanLuma : _dcHuffmanChroma; uint32 symbol = huffman->getSymbol(*bits); int dc = 0; diff --git a/video/psx_decoder.h b/video/psx_decoder.h index c96642276a5..183b6da317f 100644 --- a/video/psx_decoder.h +++ b/video/psx_decoder.h @@ -36,6 +36,7 @@ class QueuingAudioStream; } namespace Common { +template class Huffman; } @@ -105,16 +106,18 @@ private: kPlaneV = 2 }; + typedef Common::Huffman HuffmanDecoder; + uint16 _macroBlocksW, _macroBlocksH; byte *_yBuffer, *_cbBuffer, *_crBuffer; void decodeMacroBlock(Common::BitStreamMemory16LEMSB *bits, int mbX, int mbY, uint16 scale, uint16 version); void decodeBlock(Common::BitStreamMemory16LEMSB *bits, byte *block, int pitch, uint16 scale, uint16 version, PlaneType plane); void readAC(Common::BitStreamMemory16LEMSB *bits, int *block); - Common::Huffman *_acHuffman; + HuffmanDecoder *_acHuffman; int readDC(Common::BitStreamMemory16LEMSB *bits, uint16 version, PlaneType plane); - Common::Huffman *_dcHuffmanLuma, *_dcHuffmanChroma; + HuffmanDecoder *_dcHuffmanLuma, *_dcHuffmanChroma; int _lastDC[3]; void dequantizeBlock(int *coefficients, float *block, uint16 scale);