Orcus
parser_token_buffer.hpp
1/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2/*
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
6 */
7
8#ifndef INCLUDED_ORCUS_DETAIL_THREAD_PARSER_TOKEN_BUFFER_HPP
9#define INCLUDED_ORCUS_DETAIL_THREAD_PARSER_TOKEN_BUFFER_HPP
10
11#include "orcus/exception.hpp"
12
13#include <mutex>
14#include <condition_variable>
15
16namespace orcus { namespace detail { namespace thread {
17
22template<typename _TokensT>
24{
25 enum class state_type { parsing_progress, parsing_ended, parsing_aborted };
26
27 typedef _TokensT tokens_type;
28
29 mutable std::mutex m_mtx_tokens;
30 std::condition_variable m_cv_tokens_empty;
31 std::condition_variable m_cv_tokens_ready;
32
33 tokens_type m_tokens; // token buffer used to hand over tokens to the client.
34
35 size_t m_token_size_threshold;
36 const size_t m_max_token_size;
37
38 state_type m_state;
39
40 bool tokens_empty() const
41 {
42 std::lock_guard<std::mutex> lock(m_mtx_tokens);
43 return m_tokens.empty();
44 }
45
52 void wait_until_tokens_empty()
53 {
54 std::unique_lock<std::mutex> lock(m_mtx_tokens);
55 while (!m_tokens.empty() && m_state == state_type::parsing_progress)
56 m_cv_tokens_empty.wait(lock);
57
58 if (m_state == state_type::parsing_aborted)
60 }
61
62public:
63
64 parser_token_buffer(size_t min_token_size, size_t max_token_size) :
65 m_token_size_threshold(std::max<size_t>(min_token_size, 1)),
66 m_max_token_size(max_token_size),
67 m_state(state_type::parsing_progress)
68 {
69 if (m_token_size_threshold > m_max_token_size)
71 "initial token size threshold is already larger than the max token size.");
72 }
73
82 void check_and_notify(tokens_type& parser_tokens)
83 {
84 if (parser_tokens.size() < m_token_size_threshold)
85 // Still below the threshold.
86 return;
87
88 if (!tokens_empty())
89 {
90 if (m_token_size_threshold < (m_max_token_size/2))
91 {
92 // Double the threshold and continue to parse.
93 m_token_size_threshold *= 2;
94 return;
95 }
96
97 // We cannot increase the threshold any more. Wait for the
98 // client to finish.
99 wait_until_tokens_empty();
100 }
101
102 std::unique_lock<std::mutex> lock(m_mtx_tokens);
103 m_tokens.swap(parser_tokens);
104 lock.unlock();
105 m_cv_tokens_ready.notify_one();
106 }
107
116 void notify_and_finish(tokens_type& parser_tokens)
117 {
118 // Wait until the client tokens get used up.
119 wait_until_tokens_empty();
120
121 {
122 std::lock_guard<std::mutex> lock(m_mtx_tokens);
123 m_tokens.swap(parser_tokens);
124 m_state = state_type::parsing_ended;
125 }
126 m_cv_tokens_ready.notify_one();
127 }
128
129 void abort()
130 {
131 {
132 std::lock_guard<std::mutex> lock(m_mtx_tokens);
133 m_tokens.clear();
134 m_state = state_type::parsing_aborted;
135 }
136 m_cv_tokens_empty.notify_one();
137 }
138
149 bool next_tokens(tokens_type& tokens)
150 {
151 tokens.clear();
152
153 // Wait until the parser passes a new set of tokens.
154 std::unique_lock<std::mutex> lock(m_mtx_tokens);
155 while (m_tokens.empty() && m_state == state_type::parsing_progress)
156 m_cv_tokens_ready.wait(lock);
157
158 // Get the new tokens and notify the parser.
159 tokens.swap(m_tokens);
160 state_type parsing_progress = m_state; // Make a copy so that lock can be released safely.
161
162 lock.unlock();
163
164 m_cv_tokens_empty.notify_one();
165
166 return parsing_progress == state_type::parsing_progress;
167 }
168
175 size_t token_size_threshold() const
176 {
177 if (m_state == state_type::parsing_progress)
178 return 0;
179
180 return m_token_size_threshold;
181 }
182};
183
184}}}
185
186#endif
187
188/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
Definition: exception.hpp:144
Definition: parser_token_buffer.hpp:24
void notify_and_finish(tokens_type &parser_tokens)
Definition: parser_token_buffer.hpp:116
void check_and_notify(tokens_type &parser_tokens)
Definition: parser_token_buffer.hpp:82
size_t token_size_threshold() const
Definition: parser_token_buffer.hpp:175
bool next_tokens(tokens_type &tokens)
Definition: parser_token_buffer.hpp:149
Definition: exception.hpp:34
Definition: tokens.hpp:30