Make NBIS's wsq_decode_mem Thread Safe

§1. Problem
§2. Solution
§3. Result

Recently, I'm working on a project to decode WSQ files which requires to use NBIS's wsq_decode_mem function. But I found that the function is not thread-safe. After coding with AI, I made several changes to the source code to make it thread-safe.

#

Problem

The function is not thread-safe because it uses several global static variables to store the state of the decoder. When my program is running in parallel, the variables will be overwritten by different threads, causing the decoder to crash.

#

Solution

Modify source code to use thread-local variables.

In imgtools/include/wsq.himgtools/include/wsq.h insert,

c
1
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
2
#define WSQ_TLS _Thread_local
3
#elif defined(__GNUC__) || defined(__clang__)
4
#define WSQ_TLS __thread
5
#else
6
#define WSQ_TLS
7
#warning "No thread-local storage; WSQ decode won't be thread-safe."
8
#endif
1
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
2
#define WSQ_TLS _Thread_local
3
#elif defined(__GNUC__) || defined(__clang__)
4
#define WSQ_TLS __thread
5
#else
6
#define WSQ_TLS
7
#warning "No thread-local storage; WSQ decode won't be thread-safe."
8
#endif
c
1
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
2
#define WSQ_TLS _Thread_local
3
#elif defined(__GNUC__) || defined(__clang__)
4
#define WSQ_TLS __thread
5
#else
6
#define WSQ_TLS
7
#warning "No thread-local storage; WSQ decode won't be thread-safe."
8
#endif
1
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
2
#define WSQ_TLS _Thread_local
3
#elif defined(__GNUC__) || defined(__clang__)
4
#define WSQ_TLS __thread
5
#else
6
#define WSQ_TLS
7
#warning "No thread-local storage; WSQ decode won't be thread-safe."
8
#endif

In imgtools/include/wsq.himgtools/include/wsq.h, replace the global static variables with thread-local variables,

c
1
extern int debug;
2
extern WSQ_TLS QUANT_VALS quant_vals;
3
extern WSQ_TLS W_TREE w_tree[];
4
extern WSQ_TLS Q_TREE q_tree[];
5
extern WSQ_TLS DTT_TABLE dtt_table;
6
extern WSQ_TLS DQT_TABLE dqt_table;
7
extern WSQ_TLS DHT_TABLE dht_table[];
8
extern WSQ_TLS FRM_HEADER_WSQ frm_header_wsq;
9

10
/* hifilt/lofilt are read-only constants, so no need to change TLS */
11
extern float hifilt[];
12
extern float lofilt[];
1
extern int debug;
2
extern WSQ_TLS QUANT_VALS quant_vals;
3
extern WSQ_TLS W_TREE w_tree[];
4
extern WSQ_TLS Q_TREE q_tree[];
5
extern WSQ_TLS DTT_TABLE dtt_table;
6
extern WSQ_TLS DQT_TABLE dqt_table;
7
extern WSQ_TLS DHT_TABLE dht_table[];
8
extern WSQ_TLS FRM_HEADER_WSQ frm_header_wsq;
9

10
/* hifilt/lofilt are read-only constants, so no need to change TLS */
11
extern float hifilt[];
12
extern float lofilt[];
c
1
extern int debug;
2
extern WSQ_TLS QUANT_VALS quant_vals;
3
extern WSQ_TLS W_TREE w_tree[];
4
extern WSQ_TLS Q_TREE q_tree[];
5
extern WSQ_TLS DTT_TABLE dtt_table;
6
extern WSQ_TLS DQT_TABLE dqt_table;
7
extern WSQ_TLS DHT_TABLE dht_table[];
8
extern WSQ_TLS FRM_HEADER_WSQ frm_header_wsq;
9

10
/* hifilt/lofilt are read-only constants, so no need to change TLS */
11
extern float hifilt[];
12
extern float lofilt[];
1
extern int debug;
2
extern WSQ_TLS QUANT_VALS quant_vals;
3
extern WSQ_TLS W_TREE w_tree[];
4
extern WSQ_TLS Q_TREE q_tree[];
5
extern WSQ_TLS DTT_TABLE dtt_table;
6
extern WSQ_TLS DQT_TABLE dqt_table;
7
extern WSQ_TLS DHT_TABLE dht_table[];
8
extern WSQ_TLS FRM_HEADER_WSQ frm_header_wsq;
9

10
/* hifilt/lofilt are read-only constants, so no need to change TLS */
11
extern float hifilt[];
12
extern float lofilt[];

In imgtools/src/lib/wsq/globals.cimgtools/src/lib/wsq/globals.c, replace the global static definitions with thread-local variables,

c
1
#ifdef TARGET_OS
2
   WSQ_TLS QUANT_VALS quant_vals;
3
   
4
   WSQ_TLS W_TREE w_tree[W_TREELEN];
5
   
6
   WSQ_TLS Q_TREE q_tree[Q_TREELEN];
7
   
8
   WSQ_TLS DTT_TABLE dtt_table;
9
   
10
   WSQ_TLS DQT_TABLE dqt_table;
11
   
12
   WSQ_TLS DHT_TABLE dht_table[MAX_DHT_TABLES];
13
   
14
   WSQ_TLS FRM_HEADER_WSQ frm_header_wsq;
15
#else
16
   WSQ_TLS QUANT_VALS quant_vals = {};
17
   
18
   WSQ_TLS W_TREE w_tree[W_TREELEN] = {};
19
   
20
   WSQ_TLS Q_TREE q_tree[Q_TREELEN] = {};
21
   
22
   WSQ_TLS DTT_TABLE dtt_table = {};
23
   
24
   WSQ_TLS DQT_TABLE dqt_table = {};
25
   
26
   WSQ_TLS DHT_TABLE dht_table[MAX_DHT_TABLES] = {};
27
   
28
   WSQ_TLS FRM_HEADER_WSQ frm_header_wsq = {};
29
#endif
1
#ifdef TARGET_OS
2
   WSQ_TLS QUANT_VALS quant_vals;
3
   
4
   WSQ_TLS W_TREE w_tree[W_TREELEN];
5
   
6
   WSQ_TLS Q_TREE q_tree[Q_TREELEN];
7
   
8
   WSQ_TLS DTT_TABLE dtt_table;
9
   
10
   WSQ_TLS DQT_TABLE dqt_table;
11
   
12
   WSQ_TLS DHT_TABLE dht_table[MAX_DHT_TABLES];
13
   
14
   WSQ_TLS FRM_HEADER_WSQ frm_header_wsq;
15
#else
16
   WSQ_TLS QUANT_VALS quant_vals = {};
17
   
18
   WSQ_TLS W_TREE w_tree[W_TREELEN] = {};
19
   
20
   WSQ_TLS Q_TREE q_tree[Q_TREELEN] = {};
21
   
22
   WSQ_TLS DTT_TABLE dtt_table = {};
23
   
24
   WSQ_TLS DQT_TABLE dqt_table = {};
25
   
26
   WSQ_TLS DHT_TABLE dht_table[MAX_DHT_TABLES] = {};
27
   
28
   WSQ_TLS FRM_HEADER_WSQ frm_header_wsq = {};
29
#endif
c
1
#ifdef TARGET_OS
2
   WSQ_TLS QUANT_VALS quant_vals;
3
   
4
   WSQ_TLS W_TREE w_tree[W_TREELEN];
5
   
6
   WSQ_TLS Q_TREE q_tree[Q_TREELEN];
7
   
8
   WSQ_TLS DTT_TABLE dtt_table;
9
   
10
   WSQ_TLS DQT_TABLE dqt_table;
11
   
12
   WSQ_TLS DHT_TABLE dht_table[MAX_DHT_TABLES];
13
   
14
   WSQ_TLS FRM_HEADER_WSQ frm_header_wsq;
15
#else
16
   WSQ_TLS QUANT_VALS quant_vals = {};
17
   
18
   WSQ_TLS W_TREE w_tree[W_TREELEN] = {};
19
   
20
   WSQ_TLS Q_TREE q_tree[Q_TREELEN] = {};
21
   
22
   WSQ_TLS DTT_TABLE dtt_table = {};
23
   
24
   WSQ_TLS DQT_TABLE dqt_table = {};
25
   
26
   WSQ_TLS DHT_TABLE dht_table[MAX_DHT_TABLES] = {};
27
   
28
   WSQ_TLS FRM_HEADER_WSQ frm_header_wsq = {};
29
#endif
1
#ifdef TARGET_OS
2
   WSQ_TLS QUANT_VALS quant_vals;
3
   
4
   WSQ_TLS W_TREE w_tree[W_TREELEN];
5
   
6
   WSQ_TLS Q_TREE q_tree[Q_TREELEN];
7
   
8
   WSQ_TLS DTT_TABLE dtt_table;
9
   
10
   WSQ_TLS DQT_TABLE dqt_table;
11
   
12
   WSQ_TLS DHT_TABLE dht_table[MAX_DHT_TABLES];
13
   
14
   WSQ_TLS FRM_HEADER_WSQ frm_header_wsq;
15
#else
16
   WSQ_TLS QUANT_VALS quant_vals = {};
17
   
18
   WSQ_TLS W_TREE w_tree[W_TREELEN] = {};
19
   
20
   WSQ_TLS Q_TREE q_tree[Q_TREELEN] = {};
21
   
22
   WSQ_TLS DTT_TABLE dtt_table = {};
23
   
24
   WSQ_TLS DQT_TABLE dqt_table = {};
25
   
26
   WSQ_TLS DHT_TABLE dht_table[MAX_DHT_TABLES] = {};
27
   
28
   WSQ_TLS FRM_HEADER_WSQ frm_header_wsq = {};
29
#endif

In imgtools/src/lib/wsq/decoder.cimgtools/src/lib/wsq/decoder.c, replace static variables with thread-local variables in nextbits_wsqnextbits_wsq and getc_nextbits_wsqgetc_nextbits_wsq,

c
1
static WSQ_TLS unsigned char code;   /*next byte of data*/
2
static WSQ_TLS unsigned char code2;  /*stuffed byte of data*/
3
unsigned short bits, tbits;  /*bits of current data byte requested*/
4
int bits_needed;     /*additional bits required to finish request*/
5

6
                          /*used to "mask out" n number of
7
                                                        bits from data stream*/
8
static unsigned char bit_mask[9] = {0x00,0x01,0x03,0x07,0x0f,
9
                                    0x1f,0x3f,0x7f,0xff};
1
static WSQ_TLS unsigned char code;   /*next byte of data*/
2
static WSQ_TLS unsigned char code2;  /*stuffed byte of data*/
3
unsigned short bits, tbits;  /*bits of current data byte requested*/
4
int bits_needed;     /*additional bits required to finish request*/
5

6
                          /*used to "mask out" n number of
7
                                                        bits from data stream*/
8
static unsigned char bit_mask[9] = {0x00,0x01,0x03,0x07,0x0f,
9
                                    0x1f,0x3f,0x7f,0xff};
c
1
static WSQ_TLS unsigned char code;   /*next byte of data*/
2
static WSQ_TLS unsigned char code2;  /*stuffed byte of data*/
3
unsigned short bits, tbits;  /*bits of current data byte requested*/
4
int bits_needed;     /*additional bits required to finish request*/
5

6
                          /*used to "mask out" n number of
7
                                                        bits from data stream*/
8
static unsigned char bit_mask[9] = {0x00,0x01,0x03,0x07,0x0f,
9
                                    0x1f,0x3f,0x7f,0xff};
1
static WSQ_TLS unsigned char code;   /*next byte of data*/
2
static WSQ_TLS unsigned char code2;  /*stuffed byte of data*/
3
unsigned short bits, tbits;  /*bits of current data byte requested*/
4
int bits_needed;     /*additional bits required to finish request*/
5

6
                          /*used to "mask out" n number of
7
                                                        bits from data stream*/
8
static unsigned char bit_mask[9] = {0x00,0x01,0x03,0x07,0x0f,
9
                                    0x1f,0x3f,0x7f,0xff};

bit_maskbit_mask is a read-only constant, so no need to change TLS.

The codecode and code2code2 variables are used to store the next byte of data and the stuffed byte of data, respectively. After changing TLS, the variables will be stored in the thread-local storage, so they will not be overwritten by different threads.

#

Result

Multi-thread problem solved :-)

After modifying these variables, the wsq_decode_memwsq_decode_mem function is now thread-safe, so I built it and used it in my project with multiple threads, turned out to be working fine.

#Programming #Clang