// Copyright (C) 2025 Artifex Software, Inc.
//
// This file is part of MuPDF.
//
// MuPDF is free software: you can redistribute it and/or modify it under the
// terms of the GNU Affero General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// MuPDF is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
// FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
// details.
//
// You should have received a copy of the GNU Affero General Public License
// along with MuPDF. If not, see <https://www.gnu.org/licenses/agpl-3.0.en.html>
//
// Alternative licensing terms are available from the licensor.
// For commercial licensing, see <https://www.artifex.com/> or contact
// Artifex Software, Inc., 39 Mesa Street, Suite 108A, San Francisco,
// CA 94129, USA, for further information.

#include "mupdf/fitz.h"

#include <assert.h>
#include <stdlib.h>

/* #define DEBUG_WRITE_AS_PS */

/* #define DEBUG_TABLE_SPLITS */

/* #define DEBUG_TABLE_STRUCTURE */

/* #define DEBUG_TABLE_HUNT */

/* #define DEBUG_TABLE_SCORES */

/*
 * The algorithm.
 *
 *	The goal of the algorithm is to identify tables on a page.
 *	First we have to find the tables on a page, then we have to
 *	figure out where the columns/rows are, and then how the
 *	cells span them.
 *
 *	We do this as a series of steps.
 *
 *	To illustrate what's going on, let's use an example page
 *	that we can follow through all the steps.
 *
 *	+---------------------------+
 *	|                           |
 *	|      #### ## ### ##       |	<- Title
 *	|                           |
 *	|    ##### ##### #### ##    |		\
 *	|    ## ###### ###### ##    |    |
 *	|    #### ####### ######    |    |- Abstract
 *	|    ####### #### ## ###    |    |
 *	|    ### ##### ######       |   /
 *	|                           |
 *	|   #########   #########   |   2 Columns of text
 *	|   #########   #########   |
 *	|   ########    #########   |
 *	|               #########   |
 *	|   +-------+   #######     |   <- With an image on the left
 *	|   |       |               |
 *	|   |       |   ## ## # #   |   <- And a table on the right
 *	|   +-------+   ## ## # #   |
 *	|               ## ## # #   |
 *	|   #########   ## ## # #   |
 *	|   #########   ## ## # #   |
 *	|   #########   ## ## # #   |
 *	|                           |
 *	+---------------------------+
 *
 *
 * Step 1: Segmentation.
 *
 *	First, we segment the page, trying to break it down into a
 *	series of non-overlapping rectangles. We do this (in stext-boxer.c)
 *	by looking for where the content isn't. If we can identify breaks
 *	that run through the page (either from top to bottom or from left
 *	to right), then we can split the page there, and recursively consider
 *	the two halves of the page.
 *
 *	It's not a perfect algorithm, but it manages to in many cases.
 *
 *	After segmenting the above example, first we'll find the horizontal
 *	splits, giving:
 *
 *	+---------------------------+
 *	|                           |
 *	|      #### ## ### ##       |
 *	+---------------------------+
 *	|    ##### ##### #### ##    |
 *	|    ## ###### ###### ##    |
 *	|    #### ####### ######    |
 *	|    ####### #### ## ###    |
 *	|    ### ##### ######       |
 *	+---------------------------+
 *	|   #########   #########   |
 *	|   #########   #########   |
 *	|   ########    #########   |
 *	|               #########   |
 *	|   +-------+   #######     |
 *	|   |       |               |
 *	|   |       |   ## ## # #   |
 *	|   +-------+   ## ## # #   |
 *	|               ## ## # #   |
 *	|   #########   ## ## # #   |
 *	|   #########   ## ## # #   |
 *	|   #########   ## ## # #   |
 *	|                           |
 *	+---------------------------+
 *
 *	Then we'll recurse and find the vertical split between
 *	the columns:
 *
 *	+---------------------------+
 *	|                           |
 *	|      #### ## ### ##       |
 *	+---------------------------+
 *	|    ##### ##### #### ##    |
 *	|    ## ###### ###### ##    |
 *	|    #### ####### ######    |
 *	|    ####### #### ## ###    |
 *	|    ### ##### ######       |
 *	+-------------+-------------+
 *	|   ######### | #########   |
 *	|   ######### | #########   |
 *	|   ########  | #########   |
 *	|             | #########   |
 *	|   +-------+ | #######     |
 *	|   |       | |             |
 *	|   |       | | ## ## # #   |
 *	|   +-------+ | ## ## # #   |
 *	|             | ## ## # #   |
 *	|   ######### | ## ## # #   |
 *	|   ######### | ## ## # #   |
 *	|   ######### | ## ## # #   |
 *	|             |             |
 *	+-------------+-------------+
 *
 *	Then we recurse again and find the horizontal splits
 *	within the columns:
 *
 *	+---------------------------+
 *	|                           |
 *	|      #### ## ### ##       |
 *	+---------------------------+
 *	|    ##### ##### #### ##    |
 *	|    ## ###### ###### ##    |
 *	|    #### ####### ######    |
 *	|    ####### #### ## ###    |
 *	|    ### ##### ######       |
 *	+-------------+-------------+
 *	|   ######### | #########   |
 *	|   ######### | #########   |
 *	|   ########  | #########   |
 *	+-------------+ #########   |
 *	|   +-------+ | #######     |
 *	|   |       | +-------------+
 *	|   |       | | ## ## # #   |
 *	|   +-------+ | ## ## # #   |
 *	+-------------+ ## ## # #   |
 *	|   ######### | ## ## # #   |
 *	|   ######### | ## ## # #   |
 *	|   ######### | ## ## # #   |
 *	|             |             |
 *	+-------------+-------------+
 *
 *	We recurse a fixed maximum number of times (currently
 *	6, IIRC) or until we fail to find any suitable splits.
 *
 *	This completes the page segmentation step.
 *
 * Step 2: Grid finding
 *
 *	Next, we look at each of those segments and try to identify
 *	where grids might be.
 *
 *	Imagine the bottom right section of that page as
 *	a board with lego blocks on where there is text.
 *	Now imagine viewing that from the bottom of the page.
 *	The gaps between the columns of the table are where you
 *	can see through to the top between the blocks.
 *
 *	Similarly, if you view it from the side, the gaps between the
 *	rows of the page are where you can see through to the other
 *	side.
 *
 *	So, how do we code that? Well, we run through the page content
 *	(obviously, restricted to the content that falls into this
 *	segment of the page - that'll go without saying from here on
 *	in). For each bit of content, we look at the "x extent" of that
 *	content - for instance a given string might start at position
 *	10 and continue to position 100. We build a list of all these
 *	start, and stop positions, and keep them in a sorted list.
 *
 *	Then we walk this list from left to right, keeping a sum. I
 *	call this sum "wind", because it's very similar to the winding
 *	number that you get when doing scan conversion of bezier shapes.
 *
 *	wind starts out as 0. We increment it whenever we pass a 'start'
 *	position, and decrement it whenever we pass a 'stop' position.
 *	So at any given x position along the line wind tells us the
 *	number of pieces of content that overlap that x position.
 *	So wind(left) = 0 = wind(right), and wind(x) >= x for all x.
 *
 *	So, if we walk from left to right, the trace of wind might
 *	look something like:
 *
 *	             __
 *	  ___       /  \  _        __
 *	 /   \     /    \/ \     _/  \_
 *	/     \___/         \___/      \
 *
 *	The left and right edges of the table are pretty clear.
 *	The regions where wind drops to 0 represent the column dividers.
 *	The left and right hand side of those regions gives us the min
 *	and max values for that divider.
 *
 *	We can then repeat this process for Y ranges instead of X ranges
 *	to get the row dividers.
 *
 *	BUT, this only works for pure grid tables. It falls down for
 *	cases where we have merged cells (which is very common due to
 *	titles etc).
 *
 *	We can modify the algorithm slightly to allow for this.
 *
 *	Consider the following table:
 *
 *	+-----------------------------------+
 *	|  Long Table title across the top  |
 *	+---------------+---------+---------+
 *	| Name          | Result1 | Result2 |
 *	+---------------+----+----+----+----+
 *	| Homer Simpson |  1 | 23 |  4 | 56 |
 *	| Barney Gumble |  1 | 23 |  4 | 56 |
 *	| Moe           |  1 | 23 |  4 | 56 |
 *	| Apu           |  1 | 23 |  4 | 56 |
 *	| Ned Flanders  |  1 | 23 |  4 | 56 |
 *	+---------------+----+----+----+----+
 *
 *	The wind trace for that looks something like
 *	(with a certain degree of artistic license for the
 *	limitations of ascii art):
 *
 *	   ________
 *	  /        \      _   __     _   _
 *	 /          \____/ \_/  \___/ \_/ \
 *	/                                  \
 *
 *	So, the trace never quite drops back to zero in the
 *	middle due to the spanning of the top title.
 *
 *	So, instead of just looking for points where the trace
 *	drops to zero, we instead look for local minima. Each local
 *	minima represents a place where there might be a grid divider.
 *	The value of wind at such points can be considered the
 *	"uncertainty" with which there might be a divider there.
 *	Clear dividers (with a wind value of 0) have no uncertainty.
 *	Places where cells are spanned have a higher value of uncertainty.
 *
 *	The output from this step is the list of possible grid positions
 *	(X and Y), with uncertainty values.
 *
 *	We have skipped over the handling of spaces in the above
 *	description. On the one hand, we'd quite like to treat
 *	sentences as long unbroken regions of content. But sometimes
 *	we can get awkward content like:
 *
 *	P subtelomeric                 24.38 8.15  11.31 0.94   1.46
 *	                               11.08 6.93  15.34 0.00   0.73
 *	Pericentromeric region; C band 20.42 9.26  13.64 0.81   0.81
 *	                               16.50 7.03  14.45 0.17   5.15

 *	where the content is frequently sent with spaces instead of
 *	small gaps (particular in tables of digits, because numerals
 *	are often the same width even in proportional fonts).
 *
 *	To cope with this, as we send potential edges into the start/stop
 *	positions, we send 'weak' edges for the start and stops of runs
 *	of spaces. We then post process the edges to remove any local
 *	minima regions in the 'wind' values that are bounded purely by
 *	'weak' edges.
 *
 * Step 3: Cell analysis
 *
 *	So, armed with the output from step 2, we can examine each grid
 *	found. If we have W x-dividers and H y-dividers, we know we have
 *	a potential table with (W-1) x (H-1) cells in it.
 *
 *	We represent this as a W x H grid of cells, each like:
 *
 *	        .       .
 *	        .       .
 *	   . . .+-------+. . .	Each cell holds information about the
 *	        |       .	edges above, and to the left of it.
 *	        |       .
 *	        |       .
 *	   . . .+. . . .+. . .
 *	        .       .
 *	        .       .
 *
 *	h_line: Is there a horizontal divider drawn on the page that
 *	corresponds to the top of this cell (i.e. is there a cell border
 *	here?)
 *	v_line: Is there a vertical divider drawn on the page that
 *	corresponds to the left of this cell (i.e. is there a cell border
 *	here?)
 *	h_crossed: Does content cross this line (i.e. are we merged
 *	with the cell above?)
 *	v_crossed: Does content cross this line (i.e. are we merged
 *	with the cell to the left?)
 *	full: Is there any content in this cell at all?
 *
 *	We need a W x H grid of cells to represent the entire table due
 *	to the potential right and bottom edge lines. The right and
 *	bottom rows of cells should never be full, or be crossed, but
 *	it's easiest just to use a simple representation that copes with
 *	the h_line and v_line values naturally.
 *
 *	So, we start with the cells structure empty, and we run through
 *	the page content, filling in the details as we go.
 *
 *	At the end of the process, we have enough information to draw
 *	an asciiart representation of our table. It might look something
 *	like this (this comes from dotted-gridlines-tables.pdf):
 *
 *	+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *	| | |   | | |#|#|#|#| | | |
 *	+ + + + + + +v+ +v+v+ + + +
 *	| | |   | | |#|#|#|#| | | |
 *	+ + + + + + + +v+ + + + + +
 *	| |#|   | |#|#|#|#|#|#|#|#|
 *	+ +v+ + + +v+v+ +v+v+v+v+v+
 *	| |#|   |#|#|#|#|#|#|#|#|#|
 *	+ + + + +v+ + +v+ + + + + +
 *	|#|#|  #|#|#|#|#|#|#|#|#|#|
 *	+v+v+ +v+ +v+v+ +v+v+v+v+v+
 *	|#|#|  #|#|#|#|#|#|#|#|#|#|
 *	+ + + + +v+ + +v+ + + + + +
 *	| |#|   |#|#|#|#|#|#|#|#|#|
 *	+ +v+ + + +v+v+ +v+v+v+v+v+
 *	| |#|   | |#|#|#|#|#|#|#|#|
 *	+ + + + + + + +v+ + + + + +
 *	| | |   | | |#|#|#|#| | | |
 *	+ + + + + + +v+ +v+v+ + + +
 *	| | |   | | |#|#|#|#| | | |
 *	+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#  |#| | |#| | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#  |#| | |#| | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#>#|#| | |#| | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#>#|#| | |#| | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#>#|#| |#| | | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#  |#| | |#| | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#  |#| | |#| | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#  |#| | |#| | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#  |#| | |#| | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+-+
 *	  |#    |#|#|#|#|#|#|#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+-+
 *
 *	This shows where lines are detected ( - and | ),
 *	where they are crossed ( > and v) and where cells
 *	are full ( # ).
 *
 * Step 4: Row and column merging.
 *
 *	Based on the information above, we then try to merge
 *	cells and columns to simplify the table.
 *
 *	The best rules I've come up with this so far are:
 *	We can merge two adjacent columns if all the pairs of
 *	cells in the two columns are mergeable.
 *
 *	Cells are held to be mergeable or not based upon the following
 *	rules:
 *		If there is a line between 2 cells - not mergeable.
 *		else if the uncertainty between 2 cells is 0 - not mergeable.
 *		else if the line between the 2 cells is crossed - mergeable.
 *		else if strictly one of the cells is full - mergeable.
 *		else not mergeable.
 *
 *	So in the above example, column 2 (numbered from 0) can be merged
 *	with column 3.
 *
 *	This gives:
 *
 *	+-+-+-+-+-+-+-+-+-+-+-+-+
 *	| | | | | |#|#|#|#| | | |
 *	+ + + + + +v+ +v+v+ + + +
 *	| | | | | |#|#|#|#| | | |
 *	+ + + + + + +v+ + + + + +
 *	| |#| | |#|#|#|#|#|#|#|#|
 *	+ +v+ + +v+v+ +v+v+v+v+v+
 *	| |#| |#|#|#|#|#|#|#|#|#|
 *	+ + + +v+ + +v+ + + + + +
 *	|#|#|#|#|#|#|#|#|#|#|#|#|
 *	+v+v+v+ +v+v+ +v+v+v+v+v+
 *	|#|#|#|#|#|#|#|#|#|#|#|#|
 *	+ + + +v+ + +v+ + + + + +
 *	| |#| |#|#|#|#|#|#|#|#|#|
 *	+ +v+ + +v+v+ +v+v+v+v+v+
 *	| |#| | |#|#|#|#|#|#|#|#|
 *	+ + + + + + +v+ + + + + +
 *	| | | | | |#|#|#|#| | | |
 *	+ + + + + +v+ +v+v+ + + +
 *	| | | | | |#|#|#|#| | | |
 *	+-+-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#|#| | |#| | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#|#| | |#| | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#|#| | |#| | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#|#| | |#| | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#|#| |#| | | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#|#| | |#| | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#|#| | |#| | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#|#| | |#| | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+
 *	  | |#|#| | |#| | |#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+
 *	  |#  |#|#|#|#|#|#|#|#|#|
 *	+ +-+-+-+-+-+-+-+-+-+-+-+
 *
 *	We then perform the same merging process for rows as for
 *	columns - though there are no rows in the above example
 *	that can be merged.
 *
 *	You'll note that, for example, we don't merge row 0 and
 *	row 1 in the above, because we have a pair of cells that
 *	are both full without crossing.
 *
 * Step 5: Cell spanning
 *
 *	Now we actually start to output the table. We keep a 'sent_table'
 *	(a grid of W x H bools) to keep track of whether we've output
 *	the content for a given cell or not yet.
 *
 *	For each cell we reach, assuming sent_table[x,y] is false,
 *	we merge it with as many cells on the right as required,
 *	according to 'v_crossed' values (subject to not passing
 *	v_lines or uncertainty == 0's).
 *
 *	We then try to merge cells below according to 'h_crossed'
 *	values (subject to not passing h_lines or uncertainty == 0's).
 *
 *	In theory this can leave us with some cases where we'd like
 *	to merge some cells (because of crossed) and can't (because
 *	of lines or sent_table[]) values. In the absence of better
 *	cell spanning algorithms we have no choice here.
 *
 *	Then we output the contents and set sent_table[] values as
 *	appropriate.
 *
 *	If a row has no cells in it, we currently omit the TR. If/when
 *	we figure out how to indicate rowspan/colspan in stext, we can
 *	revisit that.
 */


static fz_stext_block *
add_grid_block(fz_context *ctx, fz_stext_page *page, fz_stext_block **first_block, fz_stext_block **last_block, int id)
{
	fz_stext_block *block = fz_pool_alloc(ctx, page->pool, sizeof(**first_block));
	memset(block, 0, sizeof(*block));
	block->type = FZ_STEXT_BLOCK_GRID;
	block->id = id;
	block->bbox = fz_empty_rect; /* Fixes bug 703267. */
	block->next = *first_block;
	if (*first_block)
	{
		(*first_block)->prev = block;
		assert(*last_block);
	}
	else
	{
		assert(*last_block == NULL);
		*last_block = block;
	}
	*first_block = block;
	return block;
}

static void
insert_block_before(fz_stext_block *block, fz_stext_block *before, fz_stext_page *page, fz_stext_struct *dest)
{
	if (before)
	{
		/* We have a block to insert it before, so we know it's not the last. */
		assert(dest ? (dest->first_block != NULL && dest->last_block != NULL) : (page->first_block != NULL && page->last_block != NULL));
		block->next = before;
		block->prev = before->prev;
		if (before->prev)
		{
			assert(before->prev->next == before);
			before->prev->next = block;
		}
		else if (dest)
		{
			assert(dest->first_block == before);
			dest->first_block = block;
		}
		else
		{
			assert(page->first_block == before);
			page->first_block = block;
		}
		before->prev = block;
	}
	else if (dest)
	{
		/* Will be the last block. */
		block->next = NULL;
		block->prev = dest->last_block;
		if (dest->last_block)
		{
			assert(dest->last_block->next == NULL);
			dest->last_block->next = block;
		}
		if (dest->first_block == NULL)
			dest->first_block = block;
		dest->last_block = block;
	}
	else
	{
		/* Will be the last block. */
		block->next = NULL;
		block->prev = page->last_block;
		if (page->last_block)
		{
			assert(page->last_block->next == NULL);
			page->last_block->next = block;
		}
		if (page->first_block == NULL)
			page->first_block = block;
		page->last_block = block;
	}
}

static fz_stext_struct *
add_struct_block_before(fz_context *ctx, fz_stext_block *before, fz_stext_page *page, fz_stext_struct *parent, fz_structure std, const char *raw)
{
	fz_stext_block *block;
	int idx = 0;
	size_t z;
	fz_stext_struct *newstruct;

	if (raw == NULL)
		raw = "";
	z = strlen(raw);

	/* We're going to insert a struct block. We need an idx, so walk the list */
	for (block = parent ? parent->first_block : page->first_block; block != before; block = block->next)
	{
		if (block->type == FZ_STEXT_BLOCK_STRUCT)
		{
			assert(block->u.s.index >= idx);
			idx = block->u.s.index + 1;
		}
	}
	/* So we'll add our block as idx. But all the other struct blocks that follow us need to have
	 * larger values. */

	/* Update all the subsequent structs to have a higher idx */
	if (before)
	{
		int idx2 = idx+1;
		for (block = before; block != NULL; block = block->next)
		{
			if (block->type != FZ_STEXT_BLOCK_STRUCT)
				continue;
			if (block->u.s.index > idx2)
				break;
			block->u.s.index = idx2++;
		}
	}

	/* Now make our new struct block and insert it. */
	block = fz_pool_alloc(ctx, page->pool, sizeof(*block));
	block->type = FZ_STEXT_BLOCK_STRUCT;
	block->id = 0; /* Safe because we only work on single pages */
	block->bbox = fz_empty_rect; /* Fixes bug 703267. */
	insert_block_before(block, before, page, parent);

	block->u.s.down = newstruct = fz_pool_alloc(ctx, page->pool, sizeof(*block->u.s.down) + z);
	block->u.s.index = idx;
	newstruct->parent = parent;
	newstruct->standard = std;
	memcpy(newstruct->raw, raw, z);
	newstruct->raw[z] = 0;
	newstruct->up = block;

	return newstruct;
}

typedef struct
{
	int len;
	int max;
	struct {
		uint8_t left;
		uint8_t weak;
		float pos;
		int freq;
	} *list;
} div_list;

static void
div_list_push(fz_context *ctx, div_list *div, int left, int weak, float pos)
{
	int i;

	/* FIXME: Could be bsearch. */
	for (i = 0; i < div->len; i++)
	{
		if (div->list[i].pos > pos)
			break;
		else if (div->list[i].pos == pos)
		{
			if (div->list[i].left == left)
			{
				div->list[i].freq++;
				div->list[i].weak &= weak;
				return;
			}
			if (div->list[i].left == 0)
				break;
		}
	}

	if (div->len == div->max)
	{
		int newmax = div->max * 2;
		if (newmax == 0)
			newmax = 32;
		div->list = fz_realloc(ctx, div->list, sizeof(div->list[0]) * newmax);
		div->max = newmax;
	}

	if (i < div->len)
		memmove(&div->list[i+1], &div->list[i], sizeof(div->list[0]) * (div->len - i));
	div->len++;
	div->list[i].left = left;
	div->list[i].weak = weak;
	div->list[i].pos = pos;
	div->list[i].freq = 1;
}

static fz_stext_grid_positions *
make_table_positions(fz_context *ctx, div_list *xs, float min, float max)
{
	int wind;
	fz_stext_grid_positions *pos;
	int len = xs->len;
	int i;
	int hi = 0;

	/* Count the number of edges */
	int local_min = 0;
	int edges = 2;

	if (len == 0)
		return NULL;

	assert(xs->list[0].left);
	for (i = 0; i < len; i++)
	{
		if (xs->list[i].pos >= min)
			break;
	}
	for (; i < len; i++)
	{
		if (xs->list[i].pos >= max)
			break;
		if (xs->list[i].left)
		{
			if (local_min)
				edges++;
		}
		else
			local_min = 1;
	}
	assert(!xs->list[len-1].left);

	pos = fz_malloc_flexible(ctx, fz_stext_grid_positions, list, edges);
	pos->len = edges;

	/* Copy the edges in */
	wind = 0;
	local_min = 0;
	edges = 1;
	pos->list[0].pos = min;
	pos->list[0].min = min;
	pos->list[0].max = fz_max(xs->list[0].pos, min);
	pos->list[0].uncertainty = 0;
	pos->list[0].reinforcement = 0;
#ifdef DEBUG_TABLE_HUNT
	printf("|%g ", pos->list[0].pos);
#endif
	/* Skip over entries to the left of min. */
	for (i = 0; i < len; i++)
	{
		if (xs->list[i].pos >= min)
			break;
		if (xs->list[i].left)
			wind += xs->list[i].freq;
		else
			wind -= xs->list[i].freq;
	}
	for (; i < len; i++)
	{
		if (xs->list[i].pos >= max)
			break;
		if (xs->list[i].left)
		{
			if (local_min)
			{
				pos->list[edges].min = xs->list[i-1].pos;
				pos->list[edges].max = xs->list[i].pos;
				pos->list[edges].pos = (xs->list[i-1].pos + xs->list[i].pos)/2;
				pos->list[edges].uncertainty = wind;
#ifdef DEBUG_TABLE_HUNT
				if (wind)
					printf("?%g(%d) ", pos->list[edges].pos, wind);
				else
					printf("|%g ", pos->list[edges].pos);
#endif
				edges++;
			}
			wind += xs->list[i].freq;
			if (wind > hi)
				hi = wind;
		}
		else
		{
			wind -= xs->list[i].freq;
			local_min = 1;
		}
	}
	assert(i < len || wind == 0);
	pos->list[edges].pos = max;
	pos->list[edges].min = i == 0 ? max : fz_min(xs->list[i-1].pos, max);
	pos->list[edges].max = max;
	assert(i == 0 || max >= xs->list[i-1].pos);
	pos->list[edges].uncertainty = 0;
	pos->list[edges].reinforcement = 0;
	pos->max_uncertainty = hi;
#ifdef DEBUG_TABLE_HUNT
	printf("|%g\n", pos->list[edges].pos);
#endif

	return pos;
}

static fz_stext_grid_positions *
copy_grid_positions_to_pool(fz_context *ctx, fz_stext_page *page, fz_stext_grid_positions *xs)
{
	size_t z = offsetof(fz_stext_grid_positions, list) + sizeof(xs->list[0]) * (xs->len);
	fz_stext_grid_positions *xs2 = fz_pool_alloc(ctx, page->pool, z);
	memcpy(xs2, xs, z);
	return xs2;
}

static void
sanitize_positions(fz_context *ctx, div_list *xs)
{
	int i, j, wind, changed;

#ifdef DEBUG_TABLE_HUNT
	printf("OK:\n");
	for (i = 0; i < xs->len; i++)
	{
		if (xs->list[i].left)
			printf("[");
		printf("%g(%d%s)", xs->list[i].pos, xs->list[i].freq, xs->list[i].weak ? "weak" : "");
		if (!xs->list[i].left)
			printf("]");
		printf(" ");
	}
	printf("\n");
#endif

	if (xs->len == 0)
		return;

	do
	{
		/* Now, combine runs of left and right */
		for (i = 0; i < xs->len; i++)
		{
			if (xs->list[i].left)
			{
				j = i;
				while (i < xs->len-1 && xs->list[i+1].left)
				{
					i++;
					xs->list[j].freq += xs->list[i].freq;
					xs->list[j].weak &= xs->list[i].weak;
					xs->list[i].freq = 0;
				}
			}
			else
			{
				while (i < xs->len-1 && !xs->list[i+1].left)
				{
					i++;
					xs->list[i].freq += xs->list[i-1].freq;
					xs->list[i].weak &= xs->list[i-1].weak;
					xs->list[i-1].freq = 0;
				}
			}
		}

#ifdef DEBUG_TABLE_HUNT
		printf("Shrunk:\n");
		for (i = 0; i < xs->len; i++)
		{
			if (xs->list[i].left)
				printf("[");
			printf("%g(%d%s)", xs->list[i].pos, xs->list[i].freq, xs->list[i].weak ? "weak" : "");
			if (!xs->list[i].left)
				printf("]");
			printf(" ");
		}
		printf("\n");
#endif

		/* Now remove the 0 frequency ones. */
		j = 0;
		for (i = 0; i < xs->len; i++)
		{
			if (xs->list[i].freq == 0)
				continue;
			if (i != j)
				xs->list[j] = xs->list[i];
			j++;
		}
		xs->len = j;

#ifdef DEBUG_TABLE_HUNT
		printf("Remove 0s:\n");
		for (i = 0; i < xs->len; i++)
		{
			if (xs->list[i].left)
				printf("[");
			printf("%g(%d%s)", xs->list[i].pos, xs->list[i].freq, xs->list[i].weak ? "weak" : "");
			if (!xs->list[i].left)
				printf("]");
			printf(" ");
		}
		printf("\n");
#endif

		/* Now run across looking for local minima where at least one
		 * edge is 'weak'. If the wind at that point is non-zero, then
		 * remove the weak edges from consideration and retry. */
		wind = 0;
		changed = 0;
		i = 0;
		while (1)
		{
			assert(xs->list[i].left);
			for (; xs->list[i].left; i++)
			{
				wind += xs->list[i].freq;
			}
			assert(i < xs->len);
			for (; i < xs->len && xs->list[i].left == 0; i++)
			{
				wind -= xs->list[i].freq;
			}
			if (i == xs->len)
				break;
			if (wind != 0 && (xs->list[i-1].weak || xs->list[i].weak))
			{
				int m = fz_mini(xs->list[i-1].freq, xs->list[i].freq);
				assert(m > 0);
				xs->list[i-1].freq -= m;
				xs->list[i].freq -= m;
				changed = 1;
			}
		}
	}
	while (changed);

#ifdef DEBUG_TABLE_HUNT
	printf("Compacted:\n");
	for (i = 0; i < xs->len; i++)
	{
		if (xs->list[i].left)
			printf("[");
		printf("%g(%d%s)", xs->list[i].pos, xs->list[i].freq, xs->list[i].weak ? "weak" : "");
		if (!xs->list[i].left)
			printf("]");
		printf(" ");
	}
	printf("\n");
#endif
}

/* We want to check for whether a given region contains a column
 * of justified text. We will accept some headers in this text, but
 * not JUST headers. */
static int
all_blocks_are_justified_or_headers(fz_context *ctx, fz_stext_block *block, fz_rect bounds)
{
	int just_headers = 1;

	for (; block != NULL; block = block->next)
	{
		if (fz_is_empty_rect(fz_intersect_rect(block->bbox, bounds)))
			continue;
		if (block->type == FZ_STEXT_BLOCK_STRUCT)
		{
			if (block->u.s.down == NULL)
				continue;
			if (block->u.s.down->standard == FZ_STRUCTURE_H ||
				block->u.s.down->standard == FZ_STRUCTURE_H1 ||
				block->u.s.down->standard == FZ_STRUCTURE_H2 ||
				block->u.s.down->standard == FZ_STRUCTURE_H3 ||
				block->u.s.down->standard == FZ_STRUCTURE_H4 ||
				block->u.s.down->standard == FZ_STRUCTURE_H5 ||
				block->u.s.down->standard == FZ_STRUCTURE_H6)
				continue;
			if (!all_blocks_are_justified_or_headers(ctx, block->u.s.down->first_block, bounds))
				return 0;
		}
		just_headers = 0;
		if (block->type == FZ_STEXT_BLOCK_TEXT && block->u.t.flags != FZ_STEXT_TEXT_JUSTIFY_FULL)
			return 0;
	}

	if (just_headers)
		return 0;

	return 1;
}

#define TWO_INCHES (72*2)

static void
walk_to_find_content(fz_context *ctx, div_list *xs, div_list *ys, fz_stext_block *first_block, fz_rect bounds)
{
	fz_stext_block *block;
	fz_stext_line *line;
	fz_stext_char *ch;

	for (block = first_block; block != NULL; block = block->next)
	{
		switch (block->type)
		{
		case FZ_STEXT_BLOCK_STRUCT:
			if (block->u.s.down && !fz_is_empty_rect(fz_intersect_rect(bounds, block->bbox)))
				walk_to_find_content(ctx, xs, ys, block->u.s.down->first_block, bounds);
			break;
		case FZ_STEXT_BLOCK_VECTOR:
			break;
		case FZ_STEXT_BLOCK_TEXT:
		{
			fz_rect justified_region = fz_empty_rect;
			for (line = block->u.t.first_line; line != NULL; line = line->next)
			{
				fz_rect region = fz_empty_rect;

				region.y0 = line->bbox.y0;
				region.y1 = line->bbox.y1;

				if (region.y0 < bounds.y0)
					region.y0 = bounds.y0;
				if (region.y1 > bounds.y1)
					region.y1 = bounds.y1;
				if (region.y0 >= region.y1)
					continue;

				/* Skip leading spaces. */
				for (ch = line->first_char; ch != NULL; ch = ch->next)
					if (ch->c != ' ')
						break;

				for (; ch != NULL; ch = ch->next)
				{
					if (ch->c == ' ')
					{
						/* Find the last space char in this run. */
						fz_stext_char *last_space;

						for (last_space = ch; last_space->next != NULL && last_space->next->c == ' '; last_space = last_space->next);

						/* If we're not the last char in the line (i.e. we're not a trailing space,
						 * then send a 'weak' gap for the spaces, assuming it's sane to do so). */
						if (last_space->next != NULL)
						{
							float rpos = fz_min(ch->quad.ll.x, ch->quad.ul.x);
							float lpos = fz_min(last_space->next->quad.ll.x, last_space->next->quad.ll.x);

							/* Clamp these to the bounds */
							rpos = fz_clamp(rpos, bounds.x0, bounds.x1);
							lpos = fz_clamp(lpos, bounds.x0, bounds.x1);

							/* So we have a region (rpos...lpos) to add. */
							/* This can be positioned in various different ways relative to the
							 * current region:
							 *                [region]
							 *   (rpos..lpos)                            OK
							 *        (rpos..lpos)                       OK, but adjust lpos
							 *                     (rpos..lpos)          OK, but adjust rpos
							 *                         (rpos..lpos)      OK
							 *            (rpos  ..  lpos)               OK
							 */
							if (lpos >= region.x1)
							{
								if (rpos >= region.x0 && rpos < region.x1)
									rpos = region.x1;
							}
							else if (rpos <= region.x0)
							{
								if (lpos > region.x0)
									lpos = region.x0;
							}
							else
								rpos = lpos; /* Make it an invalid region */

							if (rpos < lpos)
							{
								/* Send a weak right at the start of the spaces... */
								div_list_push(ctx, xs, 0, 1, rpos);
								/* and a weak left at the end. */
								div_list_push(ctx, xs, 1, 1, lpos);
							}

							/* Expand the region as required. */
							if (rpos < region.x0)
								region.x0 = rpos;
							if (lpos > region.x1)
								region.x1 = lpos;
						}
						/* Jump over the spaces */
						ch = last_space;
					}
					else
					{
						float lpos = fz_min(ch->quad.ll.x, ch->quad.ul.x);
						float rpos = fz_max(ch->quad.lr.x, ch->quad.ur.x);
						if (lpos < region.x0)
							region.x0 = lpos;
						if (rpos > region.x1)
							region.x1 = rpos;
					}
				}
				if (!fz_is_empty_rect(region))
				{
					div_list_push(ctx, xs, 1, 0, region.x0);
					div_list_push(ctx, xs, 0, 0, region.x1);
					/* For justified regions, we don't break after each line, but
					 * rather before/after the region as a whole. */
					if (block->u.t.flags != FZ_STEXT_TEXT_JUSTIFY_FULL)
					{
						div_list_push(ctx, ys, 1, 0, region.y0);
						div_list_push(ctx, ys, 0, 0, region.y1);
					}
					else
						justified_region = fz_union_rect(justified_region, region);
				}
			}
			if (!fz_is_empty_rect(justified_region) && block->u.t.flags == FZ_STEXT_TEXT_JUSTIFY_FULL)
			{
				div_list_push(ctx, ys, 1, 0, justified_region.y0);
				div_list_push(ctx, ys, 0, 0, justified_region.y1);
			}
			break;
		}
		}
	}
}

/* One of our datastructures (cells_t) is an array of details about the
 * cells that make up our table. It's a w * h array of cell_t's. Each
 * cell contains data on one of the cells in the table, as you'd expect.
 *
 *     .       .
 *     .       .
 * - - +-------+ - -
 *     |       .
 *     |       .
 *     |       .
 * - - + - - - + - -
 *     .       .
 *     .       .
 *
 * For any given cell, we store details about the top (lowest y coord)
 * and left (lowest x coord) edges. Specifically we store whether
 * there is a line at this position (h_line and v_line) (i.e. a drawn
 * border), and we also store whether content crosses this edge (h_crossed
 * and y_crossed). Finally, we store whether the cell has any content
 * in it at all (full).
 *
 * A table which has w positions across and h positions vertically, will
 * only really have (w-1) * (h-1) cells. We store w*h though to allow for
 * the right and bottom edges to have their lines represented.
 */

typedef struct
{
	int h_line;
	int v_line;
	int h_crossed;
	int v_crossed;
	int full;
	fz_rect content;
	float fontsize;
} cell_t;

typedef struct
{
	int w;
	int h;
	cell_t cell[FZ_FLEXIBLE_ARRAY];
} cells_t;

typedef struct
{
	cells_t *cells;
	fz_stext_grid_positions *xpos;
	fz_stext_grid_positions *ypos;
	fz_rect bounds;
	int has_background;
	float score;
} grid_walker_data;

/* This frees the contents, not the block itself! */
static void
drop_grid_walker_data(fz_context *ctx, grid_walker_data *data)
{
	if (data == NULL)
		return;
	fz_free(ctx, data->xpos);
	fz_free(ctx, data->ypos);
	fz_free(ctx, data->cells);
	data->xpos = NULL;
	data->ypos = NULL;
	data->cells = NULL;
}

static fz_stext_grid_info *
copy_grid_info_to_pool(fz_context *ctx, fz_stext_page *page, cells_t *cells)
{
	int i, n = cells->w * cells->h;
	fz_stext_grid_info *info;
	size_t z = offsetof(fz_stext_grid_info, info) + sizeof(info->info[0]) * n;
	cell_t *cell = &cells->cell[0];
	info = fz_pool_alloc(ctx, page->pool, z);

	info->w = cells->w;
	info->h = cells->h;

	for (i = 0; i < n; i++, cell++)
	{
		unsigned int flags = 0;
		if (cell->full)
			flags |= FZ_STEXT_GRID_FULL;
		if (cell->h_crossed)
			flags |= FZ_STEXT_GRID_H_CROSSED;
		if (cell->v_crossed)
			flags |= FZ_STEXT_GRID_V_CROSSED;
		if (cell->h_line)
			flags |= FZ_STEXT_GRID_T_BORDER;
		if (cell->v_line)
			flags |= FZ_STEXT_GRID_L_BORDER;
		info->info[i].flags = flags;
	}

	return info;
}

static cell_t *
get_cell(cells_t *cells, int x, int y)
{
	return &cells->cell[x + y * cells->w];
}

#if defined(DEBUG_TABLE_STRUCTURE) || defined(DEBUG_TABLE_SPLITS)
static void
asciiart_table(grid_walker_data *gd);
#endif

static fz_stext_grid_positions *
split_grid_pos(fz_context *ctx, grid_walker_data *gd, int row, int i, int early)
{
	fz_stext_grid_positions **posp = row ? &gd->ypos : &gd->xpos;
	fz_stext_grid_positions *pos = *posp;
	int n = pos->len;
	int x, y, w, h;
	cells_t *cells;

	/* Realloc the required structs */
	*posp = pos = fz_realloc_flexible(ctx, pos, fz_stext_grid_positions, list, n+1);
	cells = gd->cells = fz_realloc_flexible(ctx, gd->cells, cells_t, cell, (gd->cells->w + (1-row)) * (gd->cells->h + row));
	/* If both pass, then we're safe to shuffle the data. */

#ifdef DEBUG_TABLE_SPLITS
	printf("Before split %s %d\n", row ? "row" : "col", i);
	asciiart_table(gd);
#endif

	assert(i >= 0 && i < n);
	memmove(&pos->list[i+1], &pos->list[i], sizeof(pos->list[0]) * (n-i));
	pos->len++;

	/* Next expand the cells. Only h_line and v_line are filled in so far. */
	w = cells->w;
	h = cells->h;
	if (early && i > 0)
		i--, early = 0;
	if (row)
	{
		/* Add a row */
		cells->h = h+1;
		/* Expand the table, duplicating row i */
		memmove(&cells->cell[(i+1)*w], &cells->cell[i*w], sizeof(cells->cell[0])*(h-i)*w);

		if (early)
		{
			/* We are splitting row 0 into 0 and 1, with 0 being the new one. */
			for (x = 0; x < w; x++)
			{
				cells->cell[x].h_line = 0;
				cells->cell[x].v_line = 0;
			}
		}
		else
		{
			/* We are splitting row i into i and i+1, with i+1 being the new one. */
			/* v_lines are carried over. h_lines need to be unset. */
			for (x = 0; x < w; x++)
				cells->cell[x + (i+1)*w].h_line = 0;
		}
	}
	else
	{
		/* Add a column */
		cells->w = w+1;
		/* Expand the table, duplicating column i */
		for (y = h-1; y >= 0; y--)
		{
			for (x = w; x > i; x--)
				cells->cell[x + y*(w+1)] = cells->cell[x-1 + y*w];
			for (; x >= 0; x--)
				cells->cell[x + y*(w+1)] = cells->cell[x + y*w];
		}
		if (early)
		{
			/* We are splitting col 0 into 0 and 1, with 0 being the new one. */
			for (y = 0; y < h; y++)
			{
				cells->cell[y*(w+1)].h_line = 0;
				cells->cell[y*(w+1)].v_line = 0;
			}
		}
		else
		{
			/* h_lines are carried over. v_lines need to be reset */
			for (y = 0; y < h; y++)
				cells->cell[i+1 + y*(w+1)].v_line = 0;
		}
	}

	return pos;
}

/*	This routine finds (and reinforces) grid positions for lines.
 *
 *	If we have a thin line from (x0, y0) to (x1, y0), then we are
 *	pretty sure that y0 will be on the edge of a cell. We are less
 *	sure that x0 and x1 will match up to the edge of a cell.
 *	Stylistically some tables overrun or underrun such lines.
 *
 *	Similarly from (x0, y0) to (x0, y1), we expect x0 to be accurate
 *	but y0 and y1 less so.
 *
 *	If we have a wider rectangle, from (x0, y0) to (x1, y1) then
 *	we fully expect all sides to be accurate.
 */
static int
find_grid_pos(fz_context *ctx, grid_walker_data *gd, int row, float x, int inaccurate)
{
	const int WIGGLE_ROOM = 2;
	int i;
	fz_stext_grid_positions *pos = row ? gd->ypos : gd->xpos;

	assert(x >= pos->list[0].min && x <= pos->list[pos->len-1].max);

#ifdef DEBUG_TABLE_SPLITS
	printf("Looking for %g in %s splits:\n", x, row ? "row" : "col");
	for (i = 0; i < pos->len; i++)
	{
		printf("%d\t%g\t%g\t%g\t%d\n", i, pos->list[i].min, pos->list[i].pos, pos->list[i].max, pos->list[i].reinforcement);
	}
#endif

	while (inaccurate) /* So we can break out */
	{
		float prev = 0;

		/* If we start/finish outside the range of the table, then we
		 * want to extend the table. So ignore 'inaccurate' in this
		 * case. Match the logic below. */
		if (x < pos->list[0].min)
			break;
		if (x < pos->list[0].pos - WIGGLE_ROOM && pos->list[0].reinforcement > 0)
			break;
		if (x > pos->list[pos->len-1].max)
			break;
		if (x > pos->list[pos->len-1].pos + WIGGLE_ROOM && pos->list[pos->len-1].reinforcement > 0)
			break;

		/* Just find the closest one. No reinforcement. */
		for (i = 0; i < pos->len; i++)
		{
			if (x < pos->list[i].min)
			{
				float mid = (prev + pos->list[i].min)/2;
				if (x < mid)
					return i-1;
				return i;
			}
			prev = pos->list[i].max;
			if (x <= prev)
				return i;
		}
		assert("Never happens" == NULL);

		return -1;
	}

	for (i = 0; i < pos->len; i++)
	{
		if (x < pos->list[i].min)
		{
			/* Split i into i and i+1, and make i the new one. */
			assert(i > 0);
			if (x >= pos->list[i].min - WIGGLE_ROOM)
			{
				/* Never split into regions that would be too small. */
				return i;
			}
#ifdef DEBUG_TABLE_SPLITS
			printf("Splitting before %d\n", i);
#endif
			pos = split_grid_pos(ctx, gd, row, i, 1);
			pos->list[i-1].max = pos->list[i].min = (pos->list[i-1].max + x)/2;
			pos->list[i].pos = x;
			pos->list[i].max = pos->list[i+1].min = (pos->list[i+1].pos + x)/2;
			pos->list[i].reinforcement = 1;
#ifdef DEBUG_TABLE_SPLITS
			printf("After split\n");
			asciiart_table(gd);
#endif
			return i;
		}
		else if (x <= pos->list[i].max)
		{
			/* We are in the range for the ith divider. */
			if (i == 0)
			{
				/* Never move the first pos down, because it's been
				 * calculated to be just big enough already. But we might
				 * have to split it. */
				if (pos->list[0].pos + WIGGLE_ROOM < x)
					goto split;

				return 0;
			}
			if (i == pos->len-1)
			{
				/* Never move the last pos in, because it's been
				 * calculated to be just big enough already. But we might
				 * have to split it. */
				if (pos->list[i].pos - WIGGLE_ROOM > x)
					goto split;

				return i;
			}
			if (pos->list[i].reinforcement == 0)
			{
				/* If we've not been reinforced before, reinforce now. */
				pos->list[i].pos = x;
				pos->list[i].reinforcement = 1;
				return i;
			}
			/* We've been reinforced before. This ought to be a pretty good
			 * indication. */
			if (pos->list[i].pos - WIGGLE_ROOM < x && x < pos->list[i].pos + WIGGLE_ROOM)
			{
				/* We are a close match to the previously predicted pos
				 * value. */
				pos->list[i].pos = pos->list[i].pos * pos->list[i].reinforcement + x;
				pos->list[i].pos /= ++pos->list[i].reinforcement;
				return i;
			}
			/* We need to split i into i and i+1. */
split:
			pos = split_grid_pos(ctx, gd, row, i, pos->list[i].pos > x);
			if (pos->list[i].pos > x)
			{
				/* Make i the new one */
#ifdef DEBUG_TABLE_SPLITS
				printf("Splitting %d (early)\n", i);
#endif
				pos->list[i].pos = x;
				pos->list[i].max = pos->list[i+1].min = (pos->list[i+1].pos + x)/2;
				pos->list[i].reinforcement = 1;
#ifdef DEBUG_TABLE_SPLITS
				printf("After split\n");
				asciiart_table(gd);
#endif
				return i;
			}
			else
			{
				/* Make i+1 the new one */
#ifdef DEBUG_TABLE_SPLITS
				printf("Splitting %d (late)\n", i);
#endif
				pos->list[i+1].pos = x;
				pos->list[i].max = pos->list[i+1].min = (pos->list[i].pos + x)/2;
				pos->list[i].reinforcement = 1;
#ifdef DEBUG_TABLE_SPLITS
				printf("After split\n");
				asciiart_table(gd);
#endif
				return i+1;
			}
		}
		else if (x <= pos->list[i].max + WIGGLE_ROOM)
		{
			/* Never split into regions that would be too small. */
			return i;
		}
	}
	assert("Never happens" == NULL);

	return -1;
}

static int
find_cell_l(fz_stext_grid_positions *pos, float x)
{
	int i;

	for (i = 0; i < pos->len; i++)
		if (x < pos->list[i].pos)
			return i-1;

	return -1;
}

static int
find_cell_r(fz_stext_grid_positions *pos, float x)
{
	int i;

	for (i = 0; i < pos->len; i++)
		if (x <= pos->list[i].pos)
			return i-1;

	return -1;
}

/* Lookup the position of a horizontal line, reinforcing as appropriate. */
static void
add_h_line(fz_context *ctx, grid_walker_data *gd, float x0, float x1, float y0, float y1)
{
	float y = (y0 + y1) / 2;
	(void)find_grid_pos(ctx, gd, 1, y, 0);
}

/* Lookup the position of a vertical line, reinforcing as appropriate. */
static void
add_v_line(fz_context *ctx, grid_walker_data *gd, float y0, float y1, float x0, float x1)
{
	float x = (x0 + x1) / 2;
	(void)find_grid_pos(ctx, gd, 0, x, 0);
}

/* Lookup the position of a rectangle, reinforcing as appropriate. */
static void
add_hv_line(fz_context *ctx, grid_walker_data *gd, float x0, float x1, float y0, float y1)
{
	(void)find_grid_pos(ctx, gd, 0, x0, 0);
	(void)find_grid_pos(ctx, gd, 0, x1, 0);
	(void)find_grid_pos(ctx, gd, 1, y0, 0);
	(void)find_grid_pos(ctx, gd, 1, y1, 0);
}

/* Add a horizontal line (with no reinforcement).
 * Record which cells that was a border for. */
static void
add_h_line2(fz_context *ctx, grid_walker_data *gd, float x0, float x1, float y0, float y1)
{
	int start = find_grid_pos(ctx, gd, 0, x0, 1);
	int end = find_grid_pos(ctx, gd, 0, x1, 1);
	float y = (y0 + y1) / 2;
	int yidx = find_grid_pos(ctx, gd, 1, y, 1);
	int i;

	for (i = start; i < end; i++)
		get_cell(gd->cells, i, yidx)->h_line++;
}

/* Add a vertical line (with no reinforcement).
 * Record which cells that was a border for. */
static void
add_v_line2(fz_context *ctx, grid_walker_data *gd, float y0, float y1, float x0, float x1)
{
	int start = find_grid_pos(ctx, gd, 1, y0, 1);
	int end = find_grid_pos(ctx, gd, 1, y1, 1);
	float x = (x0 + x1) / 2;
	int xidx = find_grid_pos(ctx, gd, 0, x, 1);
	int i;

	for (i = start; i < end; i++)
		get_cell(gd->cells, xidx, i)->v_line++;
}

/* Add a rectangle (with no reinforcement).
 * Record which cells that was a border for. */
static void
add_hv_line2(fz_context *ctx, grid_walker_data *gd, float x0, float x1, float y0, float y1)
{
	int ix0 = find_grid_pos(ctx, gd, 0, x0, 1);
	int ix1 = find_grid_pos(ctx, gd, 0, x1, 1);
	int iy0 = find_grid_pos(ctx, gd, 1, y0, 1);
	int iy1 = find_grid_pos(ctx, gd, 1, y1, 1);
	int i;

	for (i = ix0; i < ix1; i++)
	{
		get_cell(gd->cells, i, iy0)->h_line++;
		get_cell(gd->cells, i, iy1)->h_line++;
	}
	for (i = iy0; i < iy1; i++)
	{
		get_cell(gd->cells, ix0, i)->v_line++;
		get_cell(gd->cells, ix1, i)->v_line++;
	}
}

/* Many PDFs with tables in them start by clearing the area of the table to white
 * by one big rectangle. Let's try and spot that. */
enum
{
	KEEP_SEARCHING = 0,
	BACKGROUND_FOUND = 1,
	NON_BACKGROUND_FOUND = 2
};

static fz_rect
whittle_background(fz_context *ctx, fz_stext_block *block0, fz_rect obounds, fz_rect bounds)
{
	fz_stext_block *block;

	for (block = block0; block != NULL; block = block->next)
	{
		if (block->type == FZ_STEXT_BLOCK_STRUCT)
		{
			if (!block->u.s.down)
				continue;
			obounds = whittle_background(ctx, block->u.s.down->first_block, obounds, bounds);
			continue;
		}
		else if (block->type == FZ_STEXT_BLOCK_VECTOR ||
			block->type == FZ_STEXT_BLOCK_TEXT)
		{
			fz_rect r = block->bbox;

			if (r.y1 >= bounds.y0 && r.y0 <= bounds.y1)
			{
				if (r.x1 <= bounds.x0)
					obounds.x0 = fz_max(obounds.x0, r.x1);
				if (r.x0 >= bounds.x1)
					obounds.x1 = fz_min(obounds.x1, r.x0);
			}
			if (r.x1 >= bounds.x0 && r.x0 <= bounds.x1)
			{
				if (r.y1 <= bounds.y0)
					obounds.y0 = fz_max(obounds.y0, r.y1);
				if (r.y0 >= bounds.y1)
					obounds.y1 = fz_min(obounds.y1, r.y0);
			}
		}
	}

	return obounds;
}

/* Can we find a fill that covers bounds, but not obounds, before any content impinges on bounds? */
static int
background_hunt(fz_context *ctx, grid_walker_data *gd, fz_stext_block *block0, fz_rect bounds, fz_rect obounds)
{
	int found;
	fz_stext_block *block;

	for (block = block0; block != NULL; block = block->next)
	{
		if (block->type == FZ_STEXT_BLOCK_STRUCT)
		{
			if (!block->u.s.down)
				continue;
			found = background_hunt(ctx, gd, block->u.s.down->first_block, bounds, obounds);
			if (found)
				return found;
			continue;
		}
		else if (block->type == FZ_STEXT_BLOCK_VECTOR)
		{
			fz_rect r = block->bbox;
			fz_rect s;

			r = fz_intersect_rect(r, gd->bounds);
			s = fz_intersect_rect(r, bounds);

			if (!fz_is_valid_rect(s))
				continue;

			/* Only rectangular && non-stroked blocks can possibly be background
			 * fills. */
			if ((block->u.v.flags & FZ_STEXT_VECTOR_IS_RECTANGLE) == 0 ||
				(block->u.v.flags & FZ_STEXT_VECTOR_IS_STROKED) != 0)
			{
				if (r.x0 < obounds.x0 || r.x1 > obounds.x1 || r.y0 < obounds.y0 || r.y1 > obounds.y1)
				{
					/* If this extends outside the outer bounds of the table, then maybe it's
					 * a background fill? Just continue. */
				}
				/* Otherwise, this might be content. Stop searching for the background fill. */
				return NON_BACKGROUND_FOUND;
			}

			/* Rectangle */
			if (r.x0 <= bounds.x0 && r.x1 >= bounds.x1 && r.y0 <= bounds.y0 && r.y1 >= bounds.y1)
			{
				/* This rectangle covers the entire table. */
				if (r.x0 < obounds.x0 || r.x1 > obounds.x1 || r.y0 < obounds.y0 || r.y1 > obounds.y1)
				{
					/* But it's too big for the table! Maybe it's a page fill? Just ignore it. */
					continue;
				}
				/* So let's extend the table. */
				gd->xpos->list[0].min = gd->xpos->list[0].pos = gd->xpos->list[0].max = r.x0;
				gd->xpos->list[gd->xpos->len-1].min = gd->xpos->list[gd->xpos->len-1].pos = gd->xpos->list[gd->xpos->len-1].max = r.x1;
				gd->ypos->list[0].min = gd->ypos->list[0].pos = gd->ypos->list[0].max = r.y0;
				gd->ypos->list[gd->ypos->len-1].min = gd->ypos->list[gd->ypos->len-1].pos = gd->ypos->list[gd->ypos->len-1].max = r.y1;

				/* And stop searching */
				return BACKGROUND_FOUND;
			}
			else
			{
				/* No background for the table found. */
				return NON_BACKGROUND_FOUND;
			}
		}
		else if (block->type == FZ_STEXT_BLOCK_TEXT)
		{
			fz_rect r = block->bbox;
			fz_rect s;

			s = fz_intersect_rect(r, gd->bounds);
			s = fz_intersect_rect(s, bounds);

			if (!fz_is_valid_rect(s))
				continue;

			return NON_BACKGROUND_FOUND;
		}
	}

	return KEEP_SEARCHING;
}

static int
walk_for_background(fz_context *ctx, grid_walker_data *gd, fz_stext_block *block0)
{
	fz_rect bounds, obounds;

	/* So the smallest covering for our table needs to be this big. */
	bounds.x0 = gd->xpos->list[0].min;
	bounds.x1 = gd->xpos->list[gd->xpos->len-1].max;
	bounds.y0 = gd->ypos->list[0].min;
	bounds.y1 = gd->ypos->list[gd->ypos->len-1].max;

	obounds = fz_infinite_rect;

	/* What is the biggest margin around this table?
	 * Start with an infinite box, and whittle it down by any content that
	 * isn't covered by the bounds. */
	obounds = whittle_background(ctx, block0, obounds, bounds);

	return background_hunt(ctx, gd, block0, bounds, obounds);
}

/* Shared internal routine with stext-boxer.c  */
fz_rect fz_collate_small_vector_run(fz_stext_block **blockp);

static void
walk_grid_lines(fz_context *ctx, grid_walker_data *gd, fz_stext_block *block)
{
	for (; block != NULL; block = block->next)
	{
		if (block->type == FZ_STEXT_BLOCK_STRUCT)
		{
			if (block->u.s.down)
				walk_grid_lines(ctx, gd, block->u.s.down->first_block);
			continue;
		}
		else if (block->type == FZ_STEXT_BLOCK_VECTOR)
		{
			fz_rect r;
			float w, h;

			/* Only process rectangle blocks. */
			if ((block->u.v.flags & FZ_STEXT_VECTOR_IS_RECTANGLE) == 0)
				continue;

			r = fz_collate_small_vector_run(&block);
			r = fz_intersect_rect(r, gd->bounds);
			if (!fz_is_valid_rect(r))
				continue;

			w = r.x1 - r.x0;
			h = r.y1 - r.y0;
			if (w > h && h < 2)
			{
				/* Thin, tall line */
				add_h_line(ctx, gd, r.x0, r.x1, r.y0, r.y1);
			}
			else if (w < h && w < 2)
			{
				/* Thin, wide line */
				add_v_line(ctx, gd, r.y0, r.y1, r.x0, r.x1);
			}
			else
			{
				/* Rectangle */
				add_hv_line(ctx, gd, r.x0, r.x1, r.y0, r.y1);
			}
		}
	}
}

static void
walk_grid_lines2(fz_context *ctx, grid_walker_data *gd, fz_stext_block *block)
{
	for (; block != NULL; block = block->next)
	{
		if (block->type == FZ_STEXT_BLOCK_STRUCT)
		{
			if (block->u.s.down)
				walk_grid_lines2(ctx, gd, block->u.s.down->first_block);
			continue;
		}
		else if (block->type == FZ_STEXT_BLOCK_VECTOR)
		{
			fz_rect r;
			float w, h;

			/* Only process rectangle blocks. */
			if ((block->u.v.flags & FZ_STEXT_VECTOR_IS_RECTANGLE) == 0)
				continue;

			r = fz_collate_small_vector_run(&block);
			r = fz_intersect_rect(r, gd->bounds);
			if (!fz_is_valid_rect(r))
				continue;

			w = r.x1 - r.x0;
			h = r.y1 - r.y0;
			if (w > h && h < 2)
			{
				/* Thin, wide line */
				add_h_line2(ctx, gd, r.x0, r.x1, r.y0, r.y1);
			}
			else if (w < h && w < 2)
			{
				/* Thin, wide line */
				add_v_line2(ctx, gd, r.y0, r.y1, r.x0, r.x1);
			}
			else
			{
				/* Rectangle */
				add_hv_line2(ctx, gd, r.x0, r.x1, r.y0, r.y1);
			}
		}
	}
}

static int
is_numeric(int c)
{
	return (c >= '0' && c <= '9');
}

static int
mark_cells_for_content(fz_context *ctx, grid_walker_data *gd, fz_rect s, float baseline, float fontsize)
{
	fz_rect r = fz_intersect_rect(gd->bounds, s);
	fz_rect line_ext = { r.x0, fz_max(r.y0, baseline-fontsize), r.x1, baseline };
	int x0, x1, y0, y1, x, y;

	/* Check for non-validity rather than empty here, as e.g.
	 * spaces are empty, and we'd still like to account for
	 * their horizontal extent. */
	if (!fz_is_valid_rect(r))
		return 0;

	x0 = find_cell_l(gd->xpos, r.x0);
	x1 = find_cell_r(gd->xpos, r.x1);
	y0 = find_cell_l(gd->ypos, r.y0);
	y1 = find_cell_r(gd->ypos, r.y1);

	if (x0 < 0 || x1 < 0 || y0 < 0 || y1 < 0)
		return 1;
	if (x0 < x1)
	{
		for (y = y0; y <= y1; y++)
			for (x = x0; x < x1; x++)
				get_cell(gd->cells, x+1, y)->v_crossed++;
	}
	if (y0 < y1)
	{
		for (y = y0; y < y1; y++)
			for (x = x0; x <= x1; x++)
				get_cell(gd->cells, x, y+1)->h_crossed++;
	}
	for (y = y0; y <= y1; y++)
		for (x = x0; x <= x1; x++)
		{
			cell_t *c = get_cell(gd->cells, x, y);
			c->full++;
			if (fontsize != 0)
			{
				c->content = fz_union_rect(c->content, line_ext);
				if (fontsize > c->fontsize)
					c->fontsize = fontsize;
			}
		}

	return 0;
}

#define IN_CELL 0
#define IN_BORDER 1
#define IN_UNKNOWN 2

#define SPLIT_MARGIN 4
static int
where_is(fz_stext_grid_positions *pos, float x, int *in)
{
	int i;

	*in = IN_UNKNOWN;

	/* If the table is empty, nothing to do. */
	if (pos->len == 0)
		return -1;

	/* If we are completely outside the table, give up. */
	if (x <= pos->list[0].pos - SPLIT_MARGIN || x >= pos->list[pos->len-1].max + SPLIT_MARGIN)
		return -1;

	for (i = 0; i < pos->len; i++)
	{
		/* Calculate the region (below..above) that counts as being
		 * on the border of position i. */
		float prev = i > 0 ? pos->list[i-1].max : pos->list[0].min;
		float next = i < pos->len-1 ? pos->list[i+1].min : pos->list[i].max;
		float below = pos->list[i].pos - SPLIT_MARGIN;
		float above = pos->list[i].pos + SPLIT_MARGIN;
		/* Find the distance half way back to the previous pos as
		 * a limit to our margin. */
		prev = (prev + pos->list[i].pos)/2;
		next = (next + pos->list[i].pos)/2;
		if (below < prev)
			below = prev;
		if (above > next)
			above = next;

		if (x < below)
		{
			*in = IN_CELL;
			return i-1;
		}
		else if (x <= above)
		{
			*in = IN_BORDER;
			return i;
		}
	}

	*in = IN_BORDER;
	return i-1;
}

enum
{
	VECTOR_IS_CONTENT = 0,
	VECTOR_IS_BORDER = 1,
	VECTOR_IS_UNKNOWN = 2,
	VECTOR_IS_IGNORABLE = 3
};

/* So a vector can either be a border, or contained
 * in some cells, or something completely else. */
static int
classify_vector(fz_context *ctx, grid_walker_data *gd, fz_rect r, int is_rect)
{
	int at_x0, at_x1, at_y0, at_y1;
	int ix0, ix1, iy0, iy1;

	r = fz_intersect_rect(r, gd->bounds);
	if (fz_is_empty_rect(r))
		return VECTOR_IS_IGNORABLE;
	ix0 = where_is(gd->xpos, r.x0, &at_x0);
	ix1 = where_is(gd->xpos, r.x1, &at_x1);
	iy0 = where_is(gd->ypos, r.y0, &at_y0);
	iy1 = where_is(gd->ypos, r.y1, &at_y1);

	/* No idea, just treat it as a border. */
	if (at_x0 == IN_UNKNOWN || at_x1 == IN_UNKNOWN || at_y0 == IN_UNKNOWN || at_y1 == IN_UNKNOWN)
		return VECTOR_IS_IGNORABLE;

	if (at_x0 == IN_BORDER && at_x1 == IN_BORDER)
	{
		/* Vector is aligned along sides of cells. */
		return is_rect ? VECTOR_IS_BORDER : VECTOR_IS_CONTENT;
	}
	if (at_y0 == IN_BORDER && at_y1 == IN_BORDER)
	{
		/* Vector is aligned along sides of cells. */
		return is_rect ? VECTOR_IS_BORDER : VECTOR_IS_CONTENT;
	}
	if (at_x0 == IN_CELL && at_x1 == IN_CELL)
	{
		/* Content within a cell (or 1d range of cells). */
		return VECTOR_IS_CONTENT;
	}
	if (at_y0 == IN_CELL && at_y1 == IN_CELL)
	{
		/* Content within a cell (or 1d range of cells). */
		return VECTOR_IS_CONTENT;
	}
	if (at_x0 == IN_BORDER && at_x1 == IN_CELL && ix0 == ix1)
	{
		return is_rect ? VECTOR_IS_BORDER : VECTOR_IS_CONTENT;
	}
	if (at_x0 == IN_CELL && at_x1 == IN_BORDER && ix0+1 == ix1)
	{
		return is_rect ? VECTOR_IS_BORDER : VECTOR_IS_CONTENT;
	}
	if (at_y0 == IN_BORDER && at_y1 == IN_CELL && iy0 == iy1)
	{
		return is_rect ? VECTOR_IS_BORDER : VECTOR_IS_CONTENT;
	}
	if (at_y0 == IN_CELL && at_y1 == IN_BORDER && iy0+1 == iy1)
	{
		return is_rect ? VECTOR_IS_BORDER : VECTOR_IS_CONTENT;
	}

	if (is_rect)
	{
		return VECTOR_IS_IGNORABLE;
	}

	/* unknown - take this as indication that this maybe isn't
	 * table. */
	return VECTOR_IS_UNKNOWN;
}

#undef IN_CELL
#undef IN_BORDER
#undef IN_UNKNOWN


/* Walk through the content, looking at how it spans our grid.
 * Record gridlines, which cells have content that cross into
 * neighbours, and which cells have any content at all.
 * Return a count of vector graphics that are found that don't
 * look plausibly like cell contents. */
static int
calculate_spanned_content(fz_context *ctx, grid_walker_data *gd, fz_stext_block *block)
{
	int duff = 0;
	fz_rect bounds = {
		gd->xpos->list[0].pos,
		gd->ypos->list[0].pos,
		gd->xpos->list[gd->xpos->len-1].pos,
		gd->ypos->list[gd->ypos->len-1].pos };

	for (; block != NULL; block = block->next)
	{
		if (block->type == FZ_STEXT_BLOCK_STRUCT)
		{
			if (block->u.s.down)
				duff += calculate_spanned_content(ctx, gd, block->u.s.down->first_block);
			continue;
		}
		else if (block->type == FZ_STEXT_BLOCK_VECTOR)
		{
			switch (classify_vector(ctx, gd, block->bbox, !!(block->u.v.flags & FZ_STEXT_VECTOR_IS_RECTANGLE)))
			{
			case VECTOR_IS_CONTENT:
				mark_cells_for_content(ctx, gd, block->bbox, 0, 0);
				break;
			case VECTOR_IS_BORDER:
			case VECTOR_IS_IGNORABLE:
				break;
			default:
				duff++;
				break;
			}
		}
		else if (block->type == FZ_STEXT_BLOCK_TEXT)
		{
			fz_stext_line *line;

			if (block->bbox.x0 >= bounds.x1 || block->bbox.y0 >= bounds.y1 ||
				block->bbox.x1 <= bounds.x0 || block->bbox.y1 <= bounds.y0)
				continue;

			for (line = block->u.t.first_line; line != NULL; line = line->next)
			{
				fz_stext_char *ch = line->first_char;
				int was_numeric = 0;

				/* Skip leading spaces */
				while (ch != NULL && ch->c == ' ')
					ch = ch->next;

				for (; ch != NULL; ch = ch->next)
				{
					if (ch->c == 32)
					{
						/* Trailing space, skip it. */
						if (ch->next == NULL)
							break;
						if (ch->next->c == 32)
						{
							/* Run of spaces. Skip 'em. */
							while (ch->next && ch->next->c == 32)
								ch = ch->next;
							was_numeric = 0;
							continue;
						}
						if (was_numeric || is_numeric(ch->next->c))
						{
							/* Single spaces around numbers are ignored. */
							was_numeric = 0;
							continue;
						}
						if (ch->flags & FZ_STEXT_SYNTHETIC_LARGE)
						{
							/* Break on large synthetic spaces */
							continue;
						}
						/* A single space. Accept it. */
						was_numeric = 0;
					}
					else
						was_numeric = is_numeric(ch->c);
					duff += mark_cells_for_content(ctx, gd, fz_rect_from_quad(ch->quad), ch->origin.y, ch->size);
				}
			}
		}
	}

	return duff;
}

static cells_t *new_cells(fz_context *ctx, int w, int h)
{
	cells_t *cells = fz_malloc_flexible(ctx, cells_t, cell, w * h);
	cells->w = w;
	cells->h = h;

	return cells;
}

#if defined(DEBUG_TABLE_STRUCTURE) || defined(DEBUG_TABLE_SPLITS)
static void
asciiart_table(grid_walker_data *gd)
{
	int w = gd->xpos->len;
	int h = gd->ypos->len;
	int x, y;

	for (x = 0; x < w; x++)
	{
		for (y = 0; y < x; y++)
			printf("| ");
		printf("(%d:%g %g %g)\n", x, gd->xpos->list[x].min, gd->xpos->list[x].pos, gd->xpos->list[x].max);
	}
	for (y = 0; y < h; y++)
	{
		for (x = 0; x < w-1; x++)
		{
			cell_t *cell = get_cell(gd->cells, x, y);
			int line = cell->h_line;
			int erase = cell->h_crossed;
			printf("+");
			if (line && !erase)
			{
				printf("-");
			}
			else if (!line && erase)
			{
				printf("v");
			}
			else if (line && erase)
			{
				printf("*");
			}
			else
			{
				printf(" ");
			}
		}
		printf("+ (%d:%g %g %g)\n", y, gd->ypos->list[y].min, gd->ypos->list[y].pos, gd->ypos->list[y].max);
		if (y == h-1)
			break;
		for (x = 0; x < w; x++)
		{
			cell_t *cell = get_cell(gd->cells, x, y);
			int line = cell->v_line;
			int erase = cell->v_crossed;
			if (line && !erase)
			{
				printf("|");
			}
			else if (!line && erase)
			{
				printf(">");
			}
			else if (line && erase)
			{
				printf("*");
			}
			else
			{
				printf(" ");
			}
			if (x < w-1)
			{
				if (cell->full)
					printf("#");
				else
					printf(" ");
			}
			else
				printf("\n");
		}
	}
}
#endif

static void
recalc_bbox(fz_stext_block *block)
{
	fz_rect bbox = fz_empty_rect;
	fz_stext_line *line;

	for (line = block->u.t.first_line; line != NULL; line = line->next)
		bbox = fz_union_rect(bbox, line->bbox);

	block->bbox = bbox;
}

static void
unlink_line_from_block(fz_stext_line *line, fz_stext_block *block)
{
	fz_stext_line *next_line = line->next;

	if (line->prev)
	{
		assert(line->prev->next == line);
		line->prev->next = next_line;
	}
	else
	{
		assert(block->u.t.first_line == line);
		block->u.t.first_line = next_line;
	}
	if (next_line)
	{
		assert(next_line->prev == line);
		next_line->prev = line->prev;
	}
	else
	{
		assert(block->u.t.last_line == line);
		block->u.t.last_line = line->prev;
	}
}

static void
append_line_to_block(fz_stext_line *line, fz_stext_block *block)
{
	if (block->u.t.last_line == NULL)
	{
		assert(block->u.t.first_line == NULL);
		block->u.t.first_line = block->u.t.last_line = line;
		line->prev = NULL;
	}
	else
	{
		assert(block->u.t.last_line->next == NULL);
		line->prev = block->u.t.last_line;
		block->u.t.last_line->next = line;
		block->u.t.last_line = line;
	}
	line->next = NULL;
}

static void
unlink_block(fz_stext_block *block, fz_stext_block **first, fz_stext_block **last)
{
	if (block->prev)
	{
		assert(block->prev->next == block);
		block->prev->next = block->next;
	}
	else
	{
		assert(*first == block);
		*first = block->next;
	}
	if (block->next)
	{
		assert(block->next->prev == block);
		block->next->prev = block->prev;
	}
	else
	{
		assert(*last == block);
		*last = block->prev;
	}
}

static int verify_stext(fz_context *ctx, fz_output *out, fz_stext_page *page, fz_stext_struct *src, const char *title)
{
	fz_stext_block *block;
	fz_stext_block **first = src ? &src->first_block : &page->first_block;
	fz_stext_block **last = src ? &src->last_block : &page->last_block;
	int max = -1;
	int fail = 0;

	if ((*first == NULL) != (*last == NULL))
	{
		if (title)
		{
			fz_write_printf(ctx, fz_stddbg(ctx), "%s\n", title);
			title = NULL;
		}
		fz_write_printf(ctx, out, "*first(%p) = %p *last(%p) = %p - should both be NULL or non-NULL\n",
			first, *first, last, *last);
		fail = 1;
	}

	for (block = *first; block != NULL; block = block->next)
	{
		fz_stext_line *line;

		if (block->prev == NULL)
		{
			if (*first != block)
			{
				if (title)
				{
					fz_write_printf(ctx, fz_stddbg(ctx), "%s\n", title);
					title = NULL;
				}
				fz_write_printf(ctx, out, "first=%p *first=%p should equal block=%p\n", first, *first, block);
				fail = 1;
			}
		}
		else if (block->prev->next != block)
		{
			if (title)
			{
				fz_write_printf(ctx, fz_stddbg(ctx), "%s\n", title);
				title = NULL;
			}
			fz_write_printf(ctx, out, "block->prev(%p)->next=%p should equal block=%p\n", block->prev, block->prev->next, block);
			fail = 1;
		}
		if (block->next == NULL)
		{
			if (*last != block)
			{
				if (title)
				{
					fz_write_printf(ctx, fz_stddbg(ctx), "%s\n", title);
					title = NULL;
				}
				fz_write_printf(ctx, out, "*last(%p)=%p should equal block=%p\n", last, *last, block);
				fail = 1;
			}
		}
		else if (block->next->prev != block)
		{
			if (title)
			{
				fz_write_printf(ctx, fz_stddbg(ctx), "%s\n", title);
				title = NULL;
			}
			fz_write_printf(ctx, out, "block->next(%p)->prev=%p should equal block=%p\n", block->next, block->next->prev, block);
			fail = 1;
		}

		if (block->type == FZ_STEXT_BLOCK_STRUCT)
		{
			if (block->u.s.down)
			{
				if (block->u.s.down->up != block)
				{
					if (title)
					{
						fz_write_printf(ctx, fz_stddbg(ctx), "%s\n", title);
						title = NULL;
					}
					fz_write_printf(ctx, out, "block->u.s.down(%p)->up should equal block(%p)\n", block->u.s.down, block->u.s.down->up, block);
					fail = 1;
				}
				if (block->u.s.down->parent != src)
				{
					if (title)
					{
						fz_write_printf(ctx, fz_stddbg(ctx), "%s\n", title);
						title = NULL;
					}
					fz_write_printf(ctx, out, "block(%p)->u.s.down(%p)->parent=%p should equal src=%p\n", block, block->u.s.down, block->u.s.down->parent, src);
					fail = 1;
				}
				fail |= verify_stext(ctx, out, page, block->u.s.down, title);
				if (block->u.s.index <= max)
				{
					if (title)
					{
						fz_write_printf(ctx, fz_stddbg(ctx), "%s\n", title);
						title = NULL;
					}
					fz_write_printf(ctx, out, "block(%p)->u.s.index=%d should be > max=%d\n", block, block->u.s.index, max);
					fail = 1;
				}
				max = block->u.s.index;
			}
			continue;
		}
		if (block->type != FZ_STEXT_BLOCK_TEXT)
			continue;
		if ((block->u.t.first_line == NULL) != (block->u.t.last_line == NULL))
		{
			if (title)
			{
				fz_write_printf(ctx, fz_stddbg(ctx), "%s\n", title);
				title = NULL;
			}
			fz_write_printf(ctx, out, "block(%p)->u.t.first_line=%p and block->u.t.last_line=(%p) should both be NULL or non-NULL\n", block, block->u.t.first_line, block->u.t.last_line);
			fail = 2;
		}
		for (line = block->u.t.first_line; line != NULL; line = line->next)
		{
			fz_stext_char *ch;

			if (line->next == NULL)
			{
				if (block->u.t.last_line != line)
				{
					if (title)
					{
						fz_write_printf(ctx, fz_stddbg(ctx), "%s\n", title);
						title = NULL;
					}
					fz_write_printf(ctx, out, "block(%p)->u.t.last_line=%p should equal line=%p\n", block, block->u.t.last_line, line);
					fail = 2;
				}
			}
			else if (line->next->prev != line)
			{
				if (title)
				{
					fz_write_printf(ctx, fz_stddbg(ctx), "%s\n", title);
					title = NULL;
				}
				fz_write_printf(ctx, out, "line->next(%p)->prev=%p should equal line=%p\n", line->next, line->next->prev, line);
				fail = 2;
			}

			if ((line->first_char == NULL) != (line->last_char == NULL))
			{
				if (title)
				{
					fz_write_printf(ctx, fz_stddbg(ctx), "%s\n", title);
					title = NULL;
				}
				fz_write_printf(ctx, out, "line(%p)->first_char=%p and line->last_char=%p should both be NULL or non-NULL\n", line, line->first_char, line->last_char);
				fail = 2;
			}

			for (ch = line->first_char; ch != NULL; ch = ch->next)
			{
				if (ch->next == NULL && line->last_char != ch)
				{
					if (title)
					{
						fz_write_printf(ctx, fz_stddbg(ctx), "%s\n", title);
						title = NULL;
					}
					fz_write_printf(ctx, out, "line(%p)->last_char=%p but last char=%p\n", line, line->last_char, ch);
					fail = 2;
				}
			}
		}
	}

	return fail;
}

void
fz_verify_stext_page(fz_context *ctx, fz_stext_page *page, const char *title)
{
	int fail;

	fail = verify_stext(ctx, fz_stddbg(ctx), page, NULL, title);
	if (fail == 0)
		return;

	fz_print_stext_page_as_xml_with_flags(ctx, fz_stddbg(ctx), page, 0, FZ_STEXT_XML_FLAGS_POINTERS | ((fail & 2) ? FZ_STEXT_XML_FLAGS_CHARS : 0));
	assert("SText verification failed" == NULL);
}

static fz_rect
move_contained_content(fz_context *ctx, fz_stext_page *page, fz_stext_struct *dest, fz_stext_struct *src, fz_rect r)
{
	fz_stext_block *before = dest ? dest->first_block : page->first_block;
	fz_stext_block **sfirst = src ? &src->first_block : &page->first_block;
	fz_stext_block **slast = src ? &src->last_block : &page->last_block;
	fz_stext_block *block, *next_block;
	int max;

	for (block = *sfirst; block != NULL; block = next_block)
	{
		fz_rect bbox = fz_intersect_rect(block->bbox, r);
		next_block = block->next;
		/* Don't use fz_is_empty_rect here, as that will exclude zero height areas like spaces. */
		if (bbox.x0 > bbox.x1 || bbox.y0 > bbox.y1)
			continue; /* Trivially excluded */
		if (bbox.x0 == block->bbox.x0 && bbox.y0 == block->bbox.y0 && bbox.x1 == block->bbox.x1 && bbox.y1 == block->bbox.y1)
		{
			/* Trivially included */
			if (block->type == FZ_STEXT_BLOCK_STRUCT && block->u.s.down)
			{
				if (block->u.s.down->standard == FZ_STRUCTURE_TD)
				{
					/* Don't copy TDs! Just copy the contents */
					move_contained_content(ctx, page, dest, block->u.s.down, r);
					continue;
				}
				block->u.s.down->parent = dest;
			}
			unlink_block(block, sfirst, slast);
			insert_block_before(block, before, page, dest);
			assert(before == block->next);
			continue;
		}
		if (block->type == FZ_STEXT_BLOCK_STRUCT)
		{
			if (block->u.s.down)
				move_contained_content(ctx, page, dest, block->u.s.down, r);
		}
		if (block->type == FZ_STEXT_BLOCK_TEXT)
		{
			/* Partially included text block */
			fz_stext_line *line, *next_line;
			fz_stext_block *newblock = NULL;

			for (line = block->u.t.first_line; line != NULL; line = next_line)
			{
				fz_rect lrect = fz_intersect_rect(line->bbox, r);
				next_line = line->next;

				/* Don't use fz_is_empty_rect here, as that will exclude zero height areas like spaces. */
				if (lrect.x0 > lrect.x1 || lrect.y0 > lrect.y1)
					continue; /* Trivial exclusion */
				if (line->bbox.x0 == lrect.x0 && line->bbox.y0 == lrect.y0 && line->bbox.x1 == lrect.x1 && line->bbox.y1 == lrect.y1)
				{
					/* Trivial inclusion */
					if (newblock == NULL)
					{
						newblock = fz_pool_alloc(ctx, page->pool, sizeof(fz_stext_block));
						insert_block_before(newblock, before, page, dest);
						newblock->type = FZ_STEXT_BLOCK_TEXT;
						newblock->id = 0;
						newblock->u.t.flags = block->u.t.flags;
						assert(before == newblock->next);
					}

					unlink_line_from_block(line, block);
					append_line_to_block(line, newblock);
				}
				else
				{
					/* Need to walk the line and just take parts */
					fz_stext_line *newline = NULL;
					fz_stext_char *ch, *ch2, *next_ch, *prev_ch = NULL;

					/* We need to be careful to use the vextent from chars that share a baseline,
					 * not the vextent of individual chars, less we cut through a line and lose
					 * half the chars. */
					/* prev_ch is the previous character left behind in the original line. */
					for (ch = line->first_char; ch != NULL; )
					{
						fz_rect crect = fz_rect_from_quad(ch->quad);
						float vmin = crect.y0;
						float vmax = crect.y1;
						float x, y;

						/* Find the next run of chars that have the same baseline, and their combined vextent. */
						for (ch2 = ch->next; ch2 != NULL; ch2 = ch2->next)
						{
							if (ch2->origin.y != ch->origin.y)
								break;
							crect = fz_rect_from_quad(ch2->quad);
							if (crect.y0 < vmin)
								vmin = crect.y0;
							if (crect.y1 > vmax)
								vmax = crect.y1;
						}

						/* So ch to ch2 (not inclusive) all have the same baseline. */
						y = (vmin + vmax)/2;

						for (; ch != ch2; ch = next_ch)
						{
							crect = fz_rect_from_quad(ch->quad);
							x = (crect.x0 + crect.x1)/2;
							next_ch = ch->next;
							if (r.x0 > x || r.x1 < x || r.y0 > y || r.y1 < y)
							{
								/* This character is outside the region
								 * we are keeping, so it is left behind. */
								prev_ch = ch;
								continue;
							}
							/* Take this char */
							if (newline == NULL)
							{
								newline = fz_pool_alloc(ctx, page->pool, sizeof(*newline));
								newline->dir = line->dir;
								newline->wmode = line->wmode;
								newline->bbox = fz_empty_rect;
							}
							/* Unlink char */
							if (prev_ch == NULL)
								line->first_char = next_ch;
							else
								prev_ch->next = next_ch;
							if (next_ch == NULL)
								line->last_char = prev_ch;
							/* Relink char */
							ch->next = NULL;
							if (newline->last_char == NULL)
								newline->first_char = ch;
							else
								newline->last_char->next = ch;
							newline->last_char = ch;
							newline->bbox = fz_union_rect(newline->bbox, crect);
						}
					}
					if (line->first_char == NULL)
					{
						/* We've removed all the chars from this line.
						 * Better remove the line too. */
						if (line->prev)
							line->prev->next = next_line;
						else
							block->u.t.first_line = next_line;
						if (next_line)
							next_line->prev = line->prev;
						else
							block->u.t.last_line = line->prev;
					}
					if (newline)
					{
						if (newblock == NULL)
						{
							newblock = fz_pool_alloc(ctx, page->pool, sizeof(fz_stext_block));

							/* Add the block onto our target list */
							insert_block_before(newblock, before, page, dest);
							newblock->type = FZ_STEXT_BLOCK_TEXT;
							newblock->id = 0;
							newblock->u.t.flags = block->u.t.flags;
							before = newblock->next;
						}
						append_line_to_block(newline, newblock);
					}
				}
			}
			if (newblock)
			{
				recalc_bbox(block);
				recalc_bbox(newblock);
			}
			if (block->u.t.first_line == NULL)
			{
				/* We've removed all the lines from the block. Should remove that too! */
				if (block->prev)
				{
					assert(block->prev->next == block);
					block->prev->next = next_block;
				}
				else
				{
					assert(*sfirst == block);
					*sfirst = block->next;
				}
				if (next_block)
				{
					assert(next_block->prev == block);
					next_block->prev = block->prev;
				}
				else
				{
					assert(*slast == block);
					*slast = block->prev;
				}
			}
		}
	}

	/* Now run through where we put blocks ensuring the indexes make sense. */
	max = 0;
	before = dest ? dest->first_block : page->first_block;
	for (block = before; block != NULL; block = block->next)
	{
		if (block->type != FZ_STEXT_BLOCK_STRUCT)
			continue;

		if (block->u.s.index < max)
			block->u.s.index = max;
		max = block->u.s.index+1;
	}

	return r;
}

typedef struct
{
	fz_stext_block *block;
	fz_stext_struct *parent;
} tree_pos;

/* This is still not perfect, but it's better! */
static tree_pos
find_table_insertion_point(fz_context *ctx, fz_rect r, tree_pos current, tree_pos best)
{
	fz_stext_block *block;

	for (block = current.block; block != NULL; block = block->next)
	{
		if (block->type == FZ_STEXT_BLOCK_STRUCT)
		{
			if (block->u.s.down != NULL && fz_is_rect_inside_rect(r, block->bbox))
			{
				tree_pos down;

				down.parent = block->u.s.down;
				down.block = block->u.s.down->first_block;
				best = find_table_insertion_point(ctx, r, down, best);
				continue;
			}
		}
		/* Is block a better precursor than best? (Or a valid precursor, if best.block == NULL) */
		if (block->bbox.y1 < r.y0 && (best.block == NULL || best.block->bbox.y1 < block->bbox.y1))
		{
			best.block = block;
			best.parent = current.parent;
		}
	}

	return best;
}

static int
tr_is_empty(fz_context *ctx, fz_stext_block *block)
{
	for (; block != NULL; block = block->next)
	{
		if (block->type != FZ_STEXT_BLOCK_STRUCT)
			return 0;
		if (!block->u.s.down)
		{
		}
		else if (block->u.s.down->standard != FZ_STRUCTURE_TD)
			return 0;
		else if (block->u.s.down->first_block != NULL)
			return 0;
	}

	return 1;
}

static int
table_is_empty(fz_context *ctx, fz_stext_block *block)
{
	for (; block != NULL; block = block->next)
	{
		if (block->type == FZ_STEXT_BLOCK_GRID)
			continue;
		if (block->type != FZ_STEXT_BLOCK_STRUCT)
			return 0;
		if (!block->u.s.down)
		{
		}
		else if (block->u.s.down->standard != FZ_STRUCTURE_TR)
			return 0;
		else if (!tr_is_empty(ctx, block->u.s.down->first_block))
			return 0;
	}

	return 1;
}

static void
tidy_td_divs(fz_context *ctx, fz_stext_struct *parent)
{
	while (1)
	{
		fz_stext_block *block = parent->first_block;
		fz_stext_block *b;

		if (block == NULL || block->next != NULL || block->type != FZ_STEXT_BLOCK_STRUCT || block->u.s.down == NULL || block->u.s.down->standard != FZ_STRUCTURE_DIV)
			return;

		parent->first_block = block->u.s.down->first_block;
		parent->last_block = block->u.s.down->last_block;

		for (b = parent->first_block; b != NULL; b = b->next)
		{
			if (b->type == FZ_STEXT_BLOCK_STRUCT && b->u.s.down)
				b->u.s.down->parent = parent;
		}
	}
}

static void
tidy_tr_divs(fz_context *ctx, fz_stext_block *block)
{
	for (; block != NULL; block = block->next)
	{
		if (block->type == FZ_STEXT_BLOCK_STRUCT && block->u.s.down && block->u.s.down->standard == FZ_STRUCTURE_TD)
			tidy_td_divs(ctx, block->u.s.down);
	}
}

static void
tidy_table_divs(fz_context *ctx, fz_stext_block *block)
{
	for (; block != NULL; block = block->next)
	{
		if (block->type == FZ_STEXT_BLOCK_GRID)
			continue;
		if (block->type == FZ_STEXT_BLOCK_STRUCT && block->u.s.down && block->u.s.down->standard == FZ_STRUCTURE_TR)
			tidy_tr_divs(ctx, block->u.s.down->first_block);
	}
}

static int
struct_is_empty(fz_context *ctx, fz_stext_block *block)
{
	for (; block != NULL; block = block->next)
	{
		if (block->type != FZ_STEXT_BLOCK_STRUCT)
			return 0;
		if (!block->u.s.down)
		{
		}
		else if (!struct_is_empty(ctx, block->u.s.down->first_block))
			return 0;
	}

	return 1;
}

static int
div_is_empty(fz_context *ctx, fz_stext_block *block)
{
	for (; block != NULL; block = block->next)
	{
		if (block->type != FZ_STEXT_BLOCK_STRUCT)
			return 0;
		if (!block->u.s.down)
		{
		}
		else if (block->u.s.down->standard == FZ_STRUCTURE_TABLE)
		{
			tidy_table_divs(ctx, block->u.s.down->first_block);
			return table_is_empty(ctx, block->u.s.down->first_block);
		}
		else if (block->u.s.down->standard != FZ_STRUCTURE_DIV)
		{
			if (!struct_is_empty(ctx, block->u.s.down->first_block))
				return 0;
		}
		else if (!div_is_empty(ctx, block->u.s.down->first_block))
			return 0;
	}

	return 1;
}

static void
tidy_orphaned_tables(fz_context *ctx, fz_stext_page *page, fz_stext_struct *parent)
{
	fz_stext_block **first_blockp = parent ? &parent->first_block : &page->first_block;
	fz_stext_block **last_blockp = parent ? &parent->last_block : &page->last_block;
	fz_stext_block *block, *next_block;

	for (block = *first_blockp; block != NULL; block = next_block)
	{
		next_block = block->next;
		if (block->type == FZ_STEXT_BLOCK_STRUCT && block->u.s.down)
		{
			if (block->u.s.down->standard == FZ_STRUCTURE_TABLE)
			{
				tidy_table_divs(ctx, block->u.s.down->first_block);
				if (table_is_empty(ctx, block->u.s.down->first_block))
				{
					/* Remove block */
					if (block->prev)
					{
						assert(block->prev->next == block);
						block->prev->next = next_block;
					}
					else
					{
						assert(*first_blockp == block);
						*first_blockp = next_block;
					}
					if (next_block)
					{
						assert(next_block->prev == block);
						next_block->prev = block->prev;
					}
					else
					{
						assert(*last_blockp == block);
						*last_blockp = block->prev;
					}
				}
				else
				{
					tidy_orphaned_tables(ctx, page, block->u.s.down);
				}
			}
			if (block->u.s.down->standard == FZ_STRUCTURE_DIV)
			{
				tidy_orphaned_tables(ctx, page, block->u.s.down);
				if (div_is_empty(ctx, block->u.s.down->first_block))
				{
					/* Remove block */
					if (block->prev)
					{
						assert(block->prev->next == block);
						block->prev->next = next_block;
					}
					else
					{
						assert(*first_blockp == block);
						*first_blockp = next_block;
					}
					if (next_block)
					{
						assert(next_block->prev == block);
						next_block->prev = block->prev;
					}
					else
					{
						assert(*last_blockp == block);
						*last_blockp = block->prev;
					}
				}
			}
		}
	}
}

/* Tiny the contents of TD up slightly; mainly this is to glue lines that may have been
 * broken apart by earlier processing back together again. */
static void
tidy_td(fz_context *ctx, fz_stext_page *page, fz_stext_struct *parent)
{
	fz_stext_block *block;
	fz_stext_block *next;
	fz_stext_char *ch;

	for (block = parent->first_block; block != NULL; block = next)
	{
		float fs0, baseline0;
		float fs1, baseline1;
		float bot, top;
		next = block->next;
		if (block->type == FZ_STEXT_BLOCK_STRUCT)
		{
			tidy_td(ctx, page, block->u.s.down);
			continue;
		}
		if (block->type != FZ_STEXT_BLOCK_TEXT)
			continue;
		if (next == NULL)
			continue;
		if (next->type != FZ_STEXT_BLOCK_TEXT)
			continue;
		/* Expect the left hand edges to match (allow for an indent) */
		if (fabsf(next->bbox.x0 - block->bbox.x0) > 36)
			continue; /* Too big a difference */
		if (next->bbox.y0 < block->bbox.y1)
			continue; /* Only if we continue downwards. */
		if (block->u.t.last_line == NULL || next->u.t.first_line == NULL)
			continue; /* Shouldn't happen... */
		fs0 = 0;
		baseline0 = 0;
		for (ch = block->u.t.last_line->first_char; ch != NULL; ch = ch->next)
		{
			if (fs0 < ch->size)
			{
				fs0 = ch->size;
				baseline0 = ch->origin.y;
			}
		}
		if (fs0 == 0)
			continue;
		fs1 = 0;
		baseline1 = 0;
		for (ch = next->u.t.first_line->first_char; ch != NULL; ch = ch->next)
		{
			if (fs1 < ch->size)
			{
				fs1 = ch->size;
				baseline1 = ch->origin.y;
			}
		}
		if (fs1 == 0)
			continue;
		/* Font sizes should mostly match */
		if (fabsf(fs0 - fs1) >= 0.5f)
			continue;
		/* This is a heuristic. Default linespacing is 120%, but we use 150% here
		 * to be a bit more forgiving. */
		bot = baseline0 + fs0/2;
		top = baseline1 - fs1;
		if (top < next->bbox.y0)
			top = next->bbox.y0;

		if (bot >= top)
		{
			/* Glue these two blocks together. */
			/* Move the lines from next onto the end of block. */
			block->u.t.last_line->next = next->u.t.first_line;
			next->u.t.first_line->prev = block->u.t.last_line;
			block->u.t.last_line = next->u.t.last_line;
			/* update the bbox */
			block->bbox = fz_union_rect(block->bbox, next->bbox);
			/* Make block point to the block after next. */
			block->next = next->next;
			/* If that's the end, then correct the parents last_block pointer */
			if (next->next == NULL)
				parent->last_block = block;
			else
				next->next->prev = block;
			/* And loop again */
			next = block;
		}
	}
}

static fz_stext_block *
transcribe_table(fz_context *ctx, grid_walker_data *gd, fz_stext_page *page, fz_stext_struct *parent)
{
	int w = gd->xpos->len;
	int h = gd->ypos->len;
	int x, y;
	char *sent_tab = fz_calloc(ctx, 1, w*(size_t)h);
	fz_stext_block **first_block = parent ? &parent->first_block : &page->first_block;
	fz_stext_struct *table, *tr, *td;
	fz_rect r;
	tree_pos insert, top;

	/* Where should we insert the table in the data? */
	r.x0 = gd->xpos->list[0].pos;
	r.x1 = gd->xpos->list[w-1].pos;
	r.y0 = gd->ypos->list[0].pos;
	r.y1 = gd->ypos->list[h-1].pos;
#ifdef DEBUG_TABLE_STRUCTURE
	printf("Region %g %g %g %g\n", r.x0, r.y0, r.x1, r.y1);
	fz_print_stext_page_as_xml(ctx, fz_stdout(ctx), page, 0);
#endif
	top.block = *first_block;
	top.parent = parent;
	insert.block = NULL;
	insert.parent = NULL;
	insert = find_table_insertion_point(ctx, r, top, insert);

	/* Convert to 'before' */
	if (insert.block)
		insert.block = insert.block->next;
	else
	{
		insert.block = parent ? parent->first_block : page->first_block;
		insert.parent = parent;
	}

	/* Make table */
	table = add_struct_block_before(ctx, insert.block, page, insert.parent, FZ_STRUCTURE_TABLE, "Table");

	/* Run through the cells, and guess at spanning. */
	for (y = 0; y < h-1; y++)
	{
		/* Have we sent this entire row before? */
		for (x = 0; x < w-1; x++)
		{
			if (!sent_tab[x+y*w])
				break;
		}
		if (x == w-1)
			continue; /* No point in sending a row with nothing in it! */

		/* Make TR */
		tr = add_struct_block_before(ctx, NULL, page, table, FZ_STRUCTURE_TR, "TR");

		for (x = 0; x < w-1; x++)
		{
			int x2, y2;
			int cellw = 1;
			int cellh = 1;

			/* Have we sent this cell already? */
			if (sent_tab[x+y*w])
				continue;

			/* Find the width of the cell */
			for (x2 = x+1; x2 < w-1; x2++)
			{
				cell_t *cell = get_cell(gd->cells, x2, y);
				if (cell->v_line)
					break; /* Can't go past a line */
				if (gd->xpos->list[x2].uncertainty == 0)
					break; /* An uncertainty of 0 is as good as a line. */
				if (!cell->v_crossed)
					break;
				cellw++;
			}
			/* Find the height of the cell */
			for (y2 = y+1; y2 < h-1; y2++)
			{
				cell_t *cell;
				int h_crossed = 0;
				if (gd->ypos->list[y2].uncertainty == 0)
					break; /* An uncertainty of 0 is as good as a line. */

				cell = get_cell(gd->cells, x, y2);
				if (cell->h_line)
					break; /* Can't extend down through a line. */
				if (cell->h_crossed)
					h_crossed = 1;
				for (x2 = x+1; x2 < x+cellw; x2++)
				{
					cell = get_cell(gd->cells, x2, y2);
					if (cell->h_line)
						break;
					if (cell->v_line)
						break; /* Can't go past a line */
					if (gd->xpos->list[x2].uncertainty == 0)
						break; /* An uncertainty of 0 is as good as a line. */
					if (!cell->v_crossed)
						break;
					if (cell->h_crossed)
						h_crossed = 1;
				}
				if (x2 == x+cellw && h_crossed)
					cellh++;
				else
					break;
			}
			/* Make TD */
			td = add_struct_block_before(ctx, NULL, page, tr, FZ_STRUCTURE_TD, "TD");
			r.x0 = gd->xpos->list[x].pos;
			r.x1 = gd->xpos->list[x+cellw].pos;
			r.y0 = gd->ypos->list[y].pos;
			r.y1 = gd->ypos->list[y+cellh].pos;
			/* Use r, not REAL contents bbox, as otherwise spanned rows
			 * can end up empty. */
			td->up->bbox = r;
			move_contained_content(ctx, page, td, parent, r);
			tidy_td(ctx, page, td);
#ifdef DEBUG_TABLE_STRUCTURE
			printf("(%d,%d) + (%d,%d)\n", x, y, cellw, cellh);
#endif
			for (y2 = y; y2 < y+cellh; y2++)
				for (x2 = x; x2 < x+cellw; x2++)
					sent_tab[x2+y2*w] = 1;
		}
		r.x0 = gd->xpos->list[0].pos;
		r.x1 = gd->xpos->list[gd->xpos->len-1].pos;
		r.y0 = gd->ypos->list[y].pos;
		r.y1 = gd->ypos->list[y+1].pos;
		tr->up->bbox = r;
		table->up->bbox = fz_union_rect(table->up->bbox, tr->up->bbox);
	}
	fz_free(ctx, sent_tab);

	{
		fz_stext_block *block;
		fz_stext_grid_positions *xps2 = copy_grid_positions_to_pool(ctx, page, gd->xpos);
		fz_stext_grid_positions *yps2 = copy_grid_positions_to_pool(ctx, page, gd->ypos);
		fz_stext_grid_info *info = copy_grid_info_to_pool(ctx, page, gd->cells);
		block = add_grid_block(ctx, page, &table->first_block, &table->last_block, table->up->id);
		block->u.b.xs = xps2;
		block->u.b.ys = yps2;
		block->u.b.info = info;
		block->bbox.x0 = block->u.b.xs->list[0].pos;
		block->bbox.y0 = block->u.b.ys->list[0].pos;
		block->bbox.x1 = block->u.b.xs->list[block->u.b.xs->len-1].pos;
		block->bbox.y1 = block->u.b.ys->list[block->u.b.ys->len-1].pos;
	}
	tidy_orphaned_tables(ctx, page, parent);

	return table->up;
}

static void
merge_column(grid_walker_data *gd, int x)
{
	int y;
	for (y = 0; y < gd->cells->h; y++)
	{
		cell_t *d = &gd->cells->cell[x + y * (gd->cells->w-1)];
		cell_t *s = &gd->cells->cell[x + y * gd->cells->w];

		if (x > 0)
			memmove(d-x, s-x, sizeof(*d) * x);
		d->full = s[0].full || s[1].full;
		d->h_crossed = s[0].h_crossed || s[1].h_crossed;
		d->h_line = s[0].h_line; /* == s[1].h_line */
		d->v_crossed = s[0].v_crossed;
		d->v_line = s[0].v_line;
		d->content = fz_union_rect(s[0].content, s[1].content);
		d->fontsize = fz_max(s[0].fontsize, s[1].fontsize);
		if (x < gd->cells->w - 2)
			memmove(d+1, s+2, sizeof(*d) * (gd->cells->w - 2 - x));
	}
	gd->cells->w--;

	if (x < gd->xpos->len - 2)
		memmove(&gd->xpos->list[x+1], &gd->xpos->list[x+2], sizeof(gd->xpos->list[0]) * (gd->xpos->len - 2 - x));
	gd->xpos->len--;
}

static void
merge_columns(grid_walker_data *gd)
{
	int x, y;

	for (x = gd->cells->w-3; x >= 0; x--)
	{
		/* Can column x be merged with column x+1? */
		/* An empty column can certainly be merged if the h_lines are the same,
		 * and there is no v_line. */
		for (y = 0; y < gd->cells->h-1; y++)
		{
			cell_t *a = get_cell(gd->cells, x, y);
			cell_t *b = get_cell(gd->cells, x+1, y);
			if (a->full || !!a->h_line != !!b->h_line || b->v_line)
				break;
		}
		if (y == gd->cells->h-1)
			goto merge_column;
		/* An empty column can certainly be merged if the h_lines are the same. */
		for (y = 0; y < gd->cells->h-1; y++)
		{
			cell_t *a = get_cell(gd->cells, x, y);
			cell_t *b = get_cell(gd->cells, x+1, y);
			if (b->full || !!a->h_line != !!b->h_line || b->v_line)
				break;
		}
		if (y == gd->cells->h-1)
			goto merge_column;
		/* We only ever want to merge columns if content crossed between them somewhere.
		 * Don't use uncertainty for this, because uncertainty doesn't allow for
		 * whitespace. */
		for (y = 0; y < gd->cells->h-1; y++)
			if (get_cell(gd->cells, x+1, y)->v_crossed == 1)
				break;
		if (y == gd->cells->h-1)
			continue;
		/* This requires all the pairs of cells in those 2 columns to be mergeable. */
		for (y = 0; y < gd->cells->h-1; y++)
		{
			cell_t *a = get_cell(gd->cells, x, y);
			cell_t *b = get_cell(gd->cells, x+1, y);
			/* If there is a divider, we can't merge. */
			if (b->v_line)
				break;
			/* If either is empty, we can merge. */
			if (!a->full || !b->full)
				continue;
			/* If we differ in h linedness, we can't merge */
			if (!!a->h_line != !!b->h_line)
				break;
			/* If both are full, we can only merge if we cross. */
			if (a->full && b->full && b->v_crossed)
				continue;
			/* Otherwise we can't merge */
			break;
		}
		if (y == gd->cells->h-1)
		{
			/* Merge the column! */
merge_column:
#ifdef DEBUG_TABLE_STRUCTURE
			printf("Merging column %d\n", x);
#endif
			merge_column(gd, x);
#ifdef DEBUG_TABLE_STRUCTURE
			asciiart_table(gd);
#endif
		}
	}
}

static void
remove_empty_column(grid_walker_data *gd, int x)
{
	int y;
	for (y = 0; y < gd->cells->h; y++)
	{
		cell_t *d = &gd->cells->cell[x + y * (gd->cells->w-1)];
		cell_t *s = &gd->cells->cell[x + y * gd->cells->w];

		if (x > 0)
			memmove(d-x, s-x, sizeof(*d) * x);
		if (x < gd->cells->w - 1)
			memmove(d, s+1, sizeof(*d) * (gd->cells->w - 1 - x));
	}
	gd->cells->w--;

	if (x < gd->xpos->len - 1)
		memmove(&gd->xpos->list[x], &gd->xpos->list[x+1], sizeof(gd->xpos->list[0]) * (gd->xpos->len - 1 - x));
	gd->xpos->len--;
}

static void
remove_empty_row(grid_walker_data *gd, int y)
{
	cell_t *d = &gd->cells->cell[y * gd->cells->w];
	cell_t *s = &gd->cells->cell[(y+1) * gd->cells->w];

	memmove(d, s, sizeof(*d) * gd->cells->w * (gd->cells->h - y - 1));
	gd->cells->h--;

	if (y < gd->ypos->len - 1)
		memmove(&gd->ypos->list[y], &gd->ypos->list[y+1], sizeof(gd->ypos->list[0]) * (gd->ypos->len - 1 - y));
	gd->ypos->len--;
}

static int
trim_empty_margins(grid_walker_data *gd)
{
	int x, y;

	/* Left column */
	while (1)
	{
		for (y = gd->cells->h-2; y >= 0; y--)
		{
			cell_t *a = get_cell(gd->cells, 0, y);
			if (a->h_line || a->v_line || a->full)
				break;
		}
		if (y >= 0)
			break;
		/* Remove the column */
		remove_empty_column(gd, 0);
#ifdef DEBUG_TABLE_STRUCTURE
		asciiart_table(gd);
#endif
		if (gd->cells->w <= 2)
			return 1;
	}

	/* Right column */
	while (1)
	{
		for (y = gd->cells->h-2; y >= 0; y--)
		{
			cell_t *a = get_cell(gd->cells, gd->cells->w-2, y);
			cell_t *b = get_cell(gd->cells, gd->cells->w-1, y);
			if (a->full || b->v_line || b->h_line)
				break;
		}
		if (y >= 0)
			break;
		/* Remove the column */
		remove_empty_column(gd, gd->cells->w-1);
#ifdef DEBUG_TABLE_STRUCTURE
		asciiart_table(gd);
#endif
		if (gd->cells->w <= 2)
			return 1;
	}

	/* Top row */
	while (1)
	{
		for (x = gd->cells->w-2; x >= 0; x--)
		{
			cell_t *a = get_cell(gd->cells, x, 0);
			if (a->h_line || a->v_line || a->full)
				break;
		}
		if (x >= 0)
			break;
		/* Remove the row */
		remove_empty_row(gd, 0);
#ifdef DEBUG_TABLE_STRUCTURE
		asciiart_table(gd);
#endif
		if (gd->cells->h <= 2)
			return 1;
	}

	/* Bottom row */
	while (1)
	{
		for (x = gd->cells->w-2; x >= 0; x--)
		{
			cell_t *a = get_cell(gd->cells, x, gd->cells->h-2);
			cell_t *b = get_cell(gd->cells, x, gd->cells->h-1);
			if (a->full || b->v_line || b->h_line)
				break;
		}
		if (x >= 0)
			break;
		/* Remove the row */
		remove_empty_row(gd, gd->cells->h-1);
#ifdef DEBUG_TABLE_STRUCTURE
		asciiart_table(gd);
#endif
		if (gd->cells->h <= 2)
			return 1;
	}

	return 0;
}

static void
merge_row(grid_walker_data *gd, int y)
{
	int x;
	int w = gd->cells->w;
	cell_t *d = &gd->cells->cell[y * w];
	for (x = 0; x < gd->cells->w-1; x++)
	{
		if (d->full == 0)
			d->full = d[w].full;
		if (d->v_crossed == 0)
			d->v_crossed = d[w].v_crossed;
		d->content = fz_union_rect(d->content, d[w].content);
		d->fontsize = fz_max(d->fontsize, d[w].fontsize);
		d++;
	}
	if (y < gd->cells->h - 2)
		memmove(d, d+w, sizeof(*d) * (gd->cells->h - 2 - y) * w);
	gd->cells->h--;

	if (y < gd->ypos->len - 2)
		memmove(&gd->ypos->list[y+1], &gd->ypos->list[y+2], sizeof(gd->ypos->list[0]) * (gd->ypos->len - 2 - y));
	gd->ypos->len--;
}

static void
merge_rows(grid_walker_data *gd)
{
	int x, y;

	for (y = gd->cells->h-3; y >= 0; y--)
	{
		/* Can row y be merged with row y+1? */
		/* An empty row can certainly be merged if the v_lines are the same,
		 * and there is no h_line. */
		for (x = 0; x < gd->cells->w-1; x++)
		{
			cell_t *a = get_cell(gd->cells, x, y);
			cell_t *b = get_cell(gd->cells, x, y+1);
			if (a->full || !!a->v_line != !!b->v_line || b->h_line)
				break;
		}
		if (x == gd->cells->w-1)
			goto merge_row;
		/* An empty row can certainly be merged if the v_lines are the same. */
		for (x = 0; x < gd->cells->w-1; x++)
		{
			cell_t *a = get_cell(gd->cells, x, y);
			cell_t *b = get_cell(gd->cells, x, y+1);
			if (b->full || !!a->v_line != !!b->v_line || b->h_line)
				break;
		}
		if (x == gd->cells->w-1)
			goto merge_row;
		/* This requires all the pairs of cells in those 2 rows to be mergeable. */
		for (x = 0; x < gd->cells->w-1; x++)
		{
			cell_t *a = get_cell(gd->cells, x, y);
			cell_t *b = get_cell(gd->cells, x, y+1);
			/* If there is a divider, we can't merge. */
			if (b->h_line)
				goto cant_merge;
			/* If either is empty, we can merge. */
			if (!a->full || !b->full)
				continue;
			/* If we differ in v linedness, we can't merge */
			if (!!a->v_line != !!b->v_line)
				goto cant_merge;
			/* If both are full... */
			if (a->full && b->full)
			{
				float fs;
				/* We can merge if we cross. */
				if (b->h_crossed)
					continue;
				/* We can merge if the line spacing looks reasonable. */
				fs = fz_max(a->fontsize, b->fontsize);
				if (a->content.y1 + fs/2 >= b->content.y0)
					continue;
			}
			/* Otherwise we can't merge */
			break;
		}
		if (x == gd->cells->w-1)
		{
			/* Merge the row! */
merge_row:
#ifdef DEBUG_TABLE_STRUCTURE
			printf("Merging row %d\n", y);
#endif
			merge_row(gd, y);
#ifdef DEBUG_TABLE_STRUCTURE
			asciiart_table(gd);
#endif
			continue;
		}

		/* OK, so we failed to be able to merge y and y+1. But if we get here, we
		 * know this wasn't because of any lines being in the way. So we can try
		 * a different set of criteria. If every non-empty cell in line y+1 is either
		 * into from line y, or crosses into line y+2, then it's probably a 'straddling'
		 * line. Just remove it. */
		if (y+2 >= gd->cells->h)
			continue;
		for (x = 0; x < gd->cells->w-1; x++)
		{
			cell_t *b = get_cell(gd->cells, x, y+1);
			cell_t *c = get_cell(gd->cells, x, y+2);
			if (!b->full)
				continue;
			if (!b->h_crossed && !c->h_crossed)
				break;
		}
		if (x == gd->cells->w-1)
		{
			/* Merge the row! */
#ifdef DEBUG_TABLE_STRUCTURE
			printf("Merging row %d (case 2)\n", y);
#endif
			merge_row(gd, y);
#ifdef DEBUG_TABLE_STRUCTURE
			asciiart_table(gd);
#endif
		}


cant_merge:
		{}
	}
}

static int
remove_bordered_empty_cells(grid_walker_data *gd)
{
	int x, y, x1, y1, x2, y2;
	int changed = 0;

	/* We are looking for a region of cells that's bordered,
	 * and empty, and where the borders don't extend...
	 *
	 *      '    '
	 *      '    '
	 * . . .'____'. . .
	 *      |    |
	 *      |    |
	 * . . .|____|. . .
	 *      '    '
	 *      '    '
	 *      '    '
	 */

	for (y = 0; y < gd->cells->h-1; y++)
	{
		for (x = 0; x < gd->cells->w-1; x++)
		{
			cell_t *u = y > 0 ? get_cell(gd->cells, x, y-1) : NULL;
			cell_t *l = x > 0 ? get_cell(gd->cells, x-1, y) : NULL;
			cell_t *c = get_cell(gd->cells, x, y);

			if (c->full)
				continue;
			if (!c->h_line || !c->v_line)
				continue;
			if (l != NULL && l->h_line)
				continue;
			if (u != NULL && u->v_line)
				continue;
			/* So c (x, y) is a reasonable top left of the bounded region. */
			for (x1 = x+1; x1 < gd->cells->w; x1++)
			{
				u = y > 0 ? get_cell(gd->cells, x1, y-1) : NULL;
				c = get_cell(gd->cells, x1, y);

				if (u != NULL && u->v_line)
					goto bad_region;
				if (c->h_line && !c->v_line)
					continue;
				if (c->h_line || !c->v_line)
					goto bad_region;
				break;
			}
			if (x1 == gd->cells->w)
				goto bad_region;
			/* If we get here, then we have the top row of a bounded region
			 * that runs from (x,y) to (x1-1, y). So, can we extend that region
			 * downwards? */
			for (y1 = y+1; y1 < gd->cells->h; y1++)
			{
				c = get_cell(gd->cells, x, y1);

				if (!c->h_line && c->v_line)
				{
					/* This could be the start of a line. */
					for (x2 = x+1; x2 < x1; x2++)
					{
						c = get_cell(gd->cells, x2, y1);
						if (c->full || c->h_line || c->v_line)
							goto bad_region;
					}
					c = get_cell(gd->cells, x1, y1);
					if (c->h_line || !c->v_line)
						goto bad_region;
					/* That line was fine! Region is extended. */
				}
				else if (c->h_line && !c->v_line)
				{
					/* This might be the bottom line of the bounded region. */
					for (x2 = x+1; x2 < x1; x2++)
					{
						c = get_cell(gd->cells, x2, y1);
						if (!c->h_line || c->v_line)
							goto bad_region;
					}
					c = get_cell(gd->cells, x1, y1);
					if (c->h_line || c->v_line)
						goto bad_region;
					break;
				}
				else
					goto bad_region;
			}
			if (y1 == gd->cells->h)
				goto bad_region;
			/* So we have a bounded region from x,y to x1-1,y1-1 */
			for (y2 = y; y2 < y1; y2++)
			{
				for (x2 = x; x2 < x1; x2++)
				{
					c = get_cell(gd->cells, x2, y2);
					c->h_line = 0;
					c->v_line = 0;
					c->full = 1;
					if (x2 > x)
						c->v_crossed = 1;
					if (y2 > y)
						c->h_crossed = 1;
				}
				c = get_cell(gd->cells, x2, y2);
				c->v_line = 0;
			}
			for (x2 = x; x2 < x1; x2++)
				get_cell(gd->cells, x2, y2)->h_line = 0;
			changed = 1;
bad_region:
			{}
		}
	}

#ifdef DEBUG_TABLE_STRUCTURE
	if (changed)
	{
		printf("Simplified empty bounded cells to give:\n");
		asciiart_table(gd);
	}
#endif

	return changed;
}

/* Is cell (x,y) plausibly part of a bordered cell (or 'super-cell'
 * allowing for spanning)? */
static int
plausibly_bordered_spanned_cell(cells_t *cells, int x, int y)
{
	int minx, miny, maxx, maxy;

	for (minx = x; minx >= 0; minx--)
	{
		cell_t *cell = get_cell(cells, minx, y);
		if (cell->v_line)
			break;
	}
	if (minx < 0)
		return 0;
	for (maxx = x+1; maxx < cells->w-1; maxx++)
	{
		cell_t *cell = get_cell(cells, maxx, y);
		if (cell->v_line)
			break;
	}
	if (maxx == cells->w)
		return 0;
	for (miny = y; miny >= 0; miny--)
	{
		cell_t *cell = get_cell(cells, x, miny);
		if (cell->h_line)
			break;
	}
	if (miny < 0)
		return 0;
	for (maxy = y+1; maxy < cells->h-1; maxy++)
	{
		cell_t *cell = get_cell(cells, x, maxy);
		if (cell->h_line)
			break;
	}
	if (maxy == cells->h)
		return 0;

	/* So we know have a plausible size for this large cell. */
	/* Now check that borders exist everywhere we expect. */
	for (x = minx; x < maxx; x++)
	{
		cell_t *cell = get_cell(cells, x, miny);
		if (!cell->h_line)
			return 0;
		cell = get_cell(cells, x, maxy);
		if (!cell->h_line)
			return 0;
	}
	for (y = miny; y < maxy; y++)
	{
		cell_t *cell = get_cell(cells, minx, y);
		if (!cell->v_line)
			return 0;
		cell = get_cell(cells, maxx, y);
		if (!cell->v_line)
			return 0;
	}

	/* And now nowhere we don't. */
	for (y = miny+1; y < maxy-1; y++)
		for (x = minx+1; x < maxx-1; x++)
		{
			cell_t *cell = get_cell(cells, x, y);
			if (cell->h_line || cell->v_line)
				return 0;
		}

	return 1;
}

/* The score for a table can be thought of as a judgement of
 * how 'awkward' a table is.
 *
 *  + Score 1 for every empty cell that isn't part of a properly
 *    bordered region, (or 0.5 if it's 'half' bordered)
 *  + Score 1 for every 'crossing' between cells.
 *
 * The score is the "average cell badness" for a table. (So,
 * a table with half unbordered cells will score the same
 * regardless of the size of the table).
 */
static float
score_table(fz_context *ctx, grid_walker_data *gd)
{
	int x, y;
	int w = gd->cells->w;
	int h = gd->cells->h;
	int score = 0;
	int num_cells = (w-1)*(h-1);

	assert(num_cells > 0);

	for (y = 0; y < h-1; y++)
	{
		for (x = 0; x < w-1; x++)
		{
			cell_t *cell = get_cell(gd->cells, x, y);
			cell_t *right = get_cell(gd->cells, x+1, y);
			cell_t *below = get_cell(gd->cells, x, y+1);
			score += cell->h_crossed + cell->v_crossed;
			if (cell->full)
			{
				/* We have content. */
			}
			else if (cell->h_line && cell->v_line && right->v_line && below->h_line)
			{
				/* The cell is trivially bordered. */
			}
			else if (plausibly_bordered_spanned_cell(gd->cells, x, y))
			{
			}
			else if (cell->h_line && below->h_line)
			{
				/* The cell is halfway bordered. */
				score += 1;
			}
			else if (cell->v_line && right->v_line)
			{
				/* The cell is halfway bordered. */
				score += 1;
			}
			else
				score += 2;
		}
	}

	return score / (float)num_cells / 2.0;
}

static void
init_cell_regions(fz_context *ctx, cells_t *cells)
{
	int i;
	int n = cells->w * cells->h;
	cell_t *c = cells->cell;

	for (i = 0; i < n; i++, c++)
		c->content = fz_empty_rect;
}


static int
find_table(fz_context *ctx, grid_walker_data *gd, fz_stext_block *content)
{
	div_list xs = { 0 };
	div_list ys = { 0 };
	int found = 0;
	int bg;

	fz_try(ctx)
	{
#ifdef DEBUG_TABLE_STRUCTURE
		printf("Hunting in: %g %g %g %g\n",
			gd->bounds.x0,
			gd->bounds.y0,
			gd->bounds.x1,
			gd->bounds.y1);
#endif
		if (all_blocks_are_justified_or_headers(ctx, content, gd->bounds))
			break;

		if (gd->xpos == NULL)
		{
#ifdef DEBUG_TABLE_STRUCTURE
			printf("Looking for grid\n");
#endif
			assert(gd->ypos == NULL);
			walk_to_find_content(ctx, &xs, &ys, content, gd->bounds);

			sanitize_positions(ctx, &xs);
			sanitize_positions(ctx, &ys);

			/* Run across the line, counting 'winding' */
			/* If we don't have at least 1 row and 1 column, give up. We'll
			 * want at least 2 each later on, but we might have a table
			 * where vectors will split the rows and columns later. */
			if (xs.len < 2 || ys.len < 2)
				break;

			gd->xpos = make_table_positions(ctx, &xs, gd->bounds.x0, gd->bounds.x1);
			gd->ypos = make_table_positions(ctx, &ys, gd->bounds.y0, gd->bounds.y1);
		}
		else
		{
			assert(gd->ypos != NULL);
		}
		gd->cells = new_cells(ctx, gd->xpos->len, gd->ypos->len);

#ifdef DEBUG_TABLE_STRUCTURE
		printf("Grid from scanning content:\n");
		asciiart_table(gd);
#endif

		bg = walk_for_background(ctx, gd, content);
		if (bg == BACKGROUND_FOUND)
			gd->has_background = 1;

		/* Walk the content looking for grid lines. These
		 * lines refine our positions. 2 passes. First pass
		 * ensures that all the lines are in the grid. Second
		 * pass then marks them. */
		walk_grid_lines(ctx, gd, content);
		walk_grid_lines2(ctx, gd, content);

		/* At this point our grid isn't going to get any larger.
		 * If we don't have at least 2 columns and 2 rows, we
		 * don't recognise it as being a table. */
		if (gd->xpos->len <= 2 || gd->ypos->len <= 2)
			break;

#ifdef DEBUG_TABLE_STRUCTURE
		printf("Grid after considering vectors:\n");
		asciiart_table(gd);
#endif

		init_cell_regions(ctx, gd->cells);
		/* Now, we walk the content looking for content that crosses
		 * these grid lines. This allows us to spot spanned cells. */
		if (calculate_spanned_content(ctx, gd, content))
			break; /* Unlikely to be a table. */

#ifdef DEBUG_TABLE_STRUCTURE
		printf("Grid after patterning content:\n");
		asciiart_table(gd);
#endif
		if (trim_empty_margins(gd))
			break; /* Trimmed away to nothing */

		/* Now, can we remove some columns or rows? i.e. have we oversegmented? */
		do
		{
			merge_columns(gd);
			merge_rows(gd);
		}
		while (remove_bordered_empty_cells(gd));

		/* Did we shrink the table so much it's not a table any more? */
		if (gd->xpos->len <= 2 || gd->ypos->len <= 2)
			break;

		found = 1;

		gd->score = score_table(ctx, gd);
#ifdef DEBUG_TABLE_STRUCTURE
		printf("%d x %d table found in: %g %g %g %g with score %g\n",
			gd->xpos->len,
			gd->ypos->len,
			gd->bounds.x0,
			gd->bounds.y0,
			gd->bounds.x1,
			gd->bounds.y1,
			gd->score);
#endif
	}
	fz_always(ctx)
	{
		fz_free(ctx, xs.list);
		fz_free(ctx, ys.list);
	}
	fz_catch(ctx)
	{
		fz_rethrow(ctx);
	}

	return found;
}

int
fz_propose_table_within_bounds(fz_context *ctx, fz_stext_page *page, fz_rect bounds, fz_stext_grid_positions **xposp, fz_stext_grid_positions **yposp)
{
	grid_walker_data gd = { 0 };
	int ret = 0;

	fz_var(ret);

	gd.bounds = bounds;

	*xposp = NULL;
	*yposp = NULL;

	fz_try(ctx)
	{
		ret = find_table(ctx, &gd, page->first_block);

		*xposp = gd.xpos;
		*yposp = gd.ypos;
		gd.xpos = NULL;
		gd.ypos = NULL;
	}
	fz_always(ctx)
		drop_grid_walker_data(ctx, &gd);
	fz_catch(ctx)
		fz_rethrow(ctx);

	return ret;
}

/**
*/
typedef struct
{
	grid_walker_data data;
	int from_raft;
	int found;
} fz_potential_table;

/**
	A list of potential tables.
*/
typedef struct
{
	int len;
	int max;
	fz_potential_table *tables;
} fz_potential_table_list;

static fz_potential_table_list *
fz_new_potential_table_list(fz_context *ctx)
{
	return fz_malloc_struct(ctx, fz_potential_table_list);
}

static void
fz_drop_potential_table_list(fz_context *ctx, fz_potential_table_list *tlist)
{
	int i;

	if (tlist == NULL)
		return;

	if (tlist->tables)
	{
		for (i = 0; i < tlist->len; i++)
			drop_grid_walker_data(ctx, &tlist->tables[i].data);
		fz_free(ctx, tlist->tables);
	}
	fz_free(ctx, tlist);
}

static void
fz_push_potential_table(fz_context *ctx, fz_potential_table_list *tlist, fz_rect r, int from_raft)
{
	if (tlist->len == tlist->max)
	{
		int n = tlist->max * 2;
		if (n == 0)
			n = 16;
		tlist->tables = fz_realloc_array(ctx, tlist->tables, n, fz_potential_table);
		tlist->max = n;
	}

	memset(&tlist->tables[tlist->len], 0, sizeof(tlist->tables[0]));
	tlist->tables[tlist->len].data.bounds = r;
	tlist->tables[tlist->len].from_raft = from_raft;
	tlist->len++;
}

static void
fz_drop_potential_table(fz_context *ctx, fz_potential_table *table)
{
	if (table)
		drop_grid_walker_data(ctx, &table->data);
}

static int
drop_contained_tables(fz_context *ctx, fz_potential_table_list *list, int lo, int hi, int at)
{
	int i, j;
	for (i = lo, j = lo; i < hi; i++)
	{
		if (i != at)
		{
			if (fz_contains_rect(list->tables[at].data.bounds, list->tables[i].data.bounds))
			{
				fz_drop_potential_table(ctx, &list->tables[i]);
				continue; /* They overlap, so drop this one. */
			}
		}
		if (i != j)
			list->tables[j] = list->tables[i];
		j++;
	}
	return j;
}

#ifdef DEBUG_TABLE_SCORES
static void
list_tables(fz_context *ctx, fz_potential_table_list *list, const char *title)
{
	int i, n;

	if (title)
		printf("%s\n", title);

	n = list->len;
	for (i = 0; i < n; i++)
	{
		printf("%d: box=(%g %g %g %g) score=%g raft=%d\n",
			i,
			list->tables[i].data.bounds.x0,
			list->tables[i].data.bounds.y0,
			list->tables[i].data.bounds.x1,
			list->tables[i].data.bounds.y1,
			list->tables[i].data.score,
			list->tables[i].from_raft);
	}
}
#endif

static void
cull_unhelpful_background_fills(fz_context *ctx, fz_stext_page *page)
{
	fz_stext_page_block_iterator iter, iter2, iter3;

	for (iter = fz_stext_page_block_iterator_begin_dfs(page);
		!fz_stext_page_block_iterator_eod_dfs(iter);
		iter = fz_stext_page_block_iterator_next_dfs(iter))
	{
		fz_stext_block *block = iter.block;
		/* We are only interested in rectangular, solid filled vectors here */
		if (block->type != FZ_STEXT_BLOCK_VECTOR ||
			(block->u.v.flags & FZ_STEXT_VECTOR_IS_RECTANGLE) == 0 ||
			(block->u.v.flags & FZ_STEXT_VECTOR_IS_STROKED) != 0 ||
			(block->u.v.argb>>24) != 0xFF)
			continue;
		/* So, that vector can 'mask' any vectors that follow it that are
		 * entirely contained within it, with the same colors. */
		/* We want to walk forward from where we currently are, but never go higher in
		 * the heirarchy. */
		iter2 = fz_stext_page_block_iterator_begin_from_dfs(page, iter.block, iter.parent);
		for (iter2 = fz_stext_page_block_iterator_next(iter2);
			!fz_stext_page_block_iterator_eod_dfs(iter2);
			iter2 = fz_stext_page_block_iterator_next_dfs(iter2))
		{
			fz_stext_block *block2 = iter2.block;
			fz_stext_block *next;
			/* We are looking for any rectangular non-stroked vector of a matching color
			 * that is contained in the original one. */
			if ((block2->type != FZ_STEXT_BLOCK_VECTOR) ||
				(block2->u.v.flags & FZ_STEXT_VECTOR_IS_RECTANGLE) == 0 ||
				(block2->u.v.flags & FZ_STEXT_VECTOR_IS_STROKED) != 0 ||
				(block2->u.v.argb != block->u.v.argb) ||
				(!fz_contains_rect(block->bbox, block2->bbox)))
					continue; /* Keep hunting */

			/* So we have a candidate vector. In order for us to be able to safely remove
			 * this vector as being 'unhelpful' then there cannot be anything in between the
			 * block, and block2 that overlaps block2. */
			iter3 = fz_stext_page_block_iterator_begin_from_dfs(page, iter.block, iter.parent);
			for (iter3 = fz_stext_page_block_iterator_next(iter3);
				iter2.block != iter3.block;
				iter3 = fz_stext_page_block_iterator_next_dfs(iter3))
			{
				fz_stext_block *block3 = iter3.block;
				if (fz_overlaps_rect(block2->bbox, block3->bbox))
					break;
			}
			if (iter2.block != iter3.block)
				continue;

			/* Remove this vector. It's masked by the outside one. */
			next = block2->next;
			block2->prev->next = next;
			if (next)
				next->prev = block2->prev;
			else if (iter.parent)
				iter.parent->last_block = block2->prev;
			else
				iter.page->last_block = block2->prev;
		}
	}
}

static float
scaled_table_score(fz_potential_table_list *list, int k)
{
	float score = list->tables[k].data.score;
	float w = list->tables[k].data.bounds.x1 - list->tables[k].data.bounds.x0;
	float h = list->tables[k].data.bounds.y1 - list->tables[k].data.bounds.y0;

	if (w <= 0 || h <= 0)
		return 999999; /* Zero area tables are bad choices. */
	return score / w / h;
}

static int
table_size_cmp(const void *a_, const void *b_)
{
	const fz_potential_table *a = a_;
	const fz_potential_table *b = b_;
	float w_a = (a->data.bounds.x1 - a->data.bounds.x0);
	float h_a = (a->data.bounds.y1 - a->data.bounds.y0);
	float area_a = (w_a > 0 && h_a > 0) ? w_a * h_a : 0;
	float w_b = (b->data.bounds.x1 - b->data.bounds.x0);
	float h_b = (b->data.bounds.y1 - b->data.bounds.y0);
	float area_b = (w_b > 0 && h_b > 0) ? w_b * h_b : 0;

	if (area_a < area_b)
		return -1;
	else if (area_a > area_b)
		return 1;
	/* Some extra logic to ensure stability */
	else if (a < b)
		return -1;
	else if (a > b)
		return 1;
	return 0;
}

/* Takes ownership of list, and frees before return. */
static fz_stext_block *
hunt_potential_tables(fz_context *ctx, fz_stext_page *page, fz_potential_table_list *list, float limit)
{
	int i, j, k, n;
	fz_stext_block *last = NULL;

	if (!list)
		return NULL;

	fz_try(ctx)
	{
		cull_unhelpful_background_fills(ctx, page);

#ifdef DEBUG_TABLE_SCORES
		list_tables(ctx, list, "pre find_table");
#endif

		/* Look for tables in all the possible positions */
		n = list->len;
		for (i = 0; i < n; i++)
			list->tables[i].found = find_table(ctx, &list->tables[i].data, page->first_block);

		/* Cull the tables that weren't found. */
		n = list->len;
		for (j = 0, i = 0; i < n; i++)
		{
			if (!list->tables[i].found)
			{
				fz_drop_potential_table(ctx, &list->tables[i]);
				continue;
			}
			if (i != j)
				list->tables[j] = list->tables[i];
			j++;
		}
		list->len = j;

#ifdef DEBUG_TABLE_SCORES
		list_tables(ctx, list, "pre raft cull");
#endif

		/* Remove any tables that are masked by tables that came from rafts. */
		n = list->len;
		for (j = 0, i = 0; i < n; i++)
		{
			if (!list->tables[i].from_raft)
			{
				/* Is 'i' masked by a from_raft one? */
				for (k = 0; k < n; k++)
				{
					if (!list->tables[k].from_raft)
						continue;
					if (fz_contains_rect(list->tables[k].data.bounds, list->tables[i].data.bounds))
						break;
				}
				if (k != n)
				{
					fz_drop_potential_table(ctx, &list->tables[i]);
					continue;
				}
			}
#ifndef NDEBUG
			else
			{
				/* i is from a raft. No other one from a raft can intersect it. */
				for (k = 0; k < n; k++)
				{
					if (i == k)
						continue;
					if (!list->tables[k].from_raft)
						continue;
					assert(fz_is_empty_rect(fz_intersect_rect(list->tables[i].data.bounds, list->tables[k].data.bounds)));
				}
			}
#endif
			if (i != j)
				list->tables[j] = list->tables[i];
			j++;
		}
		list->len = j;

#ifdef DEBUG_TABLE_SCORES
		list_tables(ctx, list, "pre sort");
#endif
		if (list->len > 0)
			qsort(&list->tables[0], list->len, sizeof(list->tables[0]), table_size_cmp);
#ifdef DEBUG_TABLE_SCORES
		list_tables(ctx, list, "pre cull");
#endif

		/* Further cull the tables. */
		/* For each table, look to see if it contains others. Choose between this table
		 * or the ones it contains */
		n = list->len;
		for (j = 0, i = 0; i < n; i++)
		{
			/* At any point in this loop: 0..j, and i to n are valid. */
			float overlapped_score = 0;
			int num_overlaps = 0;
			for (k = 0; k < n; k++)
			{
				if (k == j)
				{
					k = i+1;
					if (k == n)
						break;
				}
				if (fz_contains_rect(list->tables[i].data.bounds, list->tables[k].data.bounds))
				{
					/* We contain table k. */
					overlapped_score += scaled_table_score(list, k);
					num_overlaps++;
				}
			}
			/* If we don't overlap anything, nothing to worry about. Just keep us. */
			if (num_overlaps)
			{
				if (overlapped_score <= scaled_table_score(list, i))
				{
					/* The "badness" for the subtables is less than the "badness" for this table. Drop this table. */
					fz_drop_potential_table(ctx, &list->tables[i]);
					continue;
				}
				/* Drop all the tables we contain. */
				j = drop_contained_tables(ctx, list, 0, j, i);
				n = list->len = drop_contained_tables(ctx, list, i+1, n, i);
			}
			if (i != j)
				list->tables[j] = list->tables[i];
			j++;
		}
		list->len = j;

#ifdef DEBUG_TABLE_SCORES
		list_tables(ctx, list, "post contained cull");
#endif

		/* We need to cull to the point where no 2 tables overlap. At this point
		 * we just keep the better of any 2 intersecting tables. */
		n = list->len;
		for (j = 0, i = 0; i < n; i++)
		{
			/* At any point in this loop: 0..j, and i to n are valid. */
			for (k = 0; k < n; k++)
			{
				if (k == j)
				{
					k = i+1;
					if (k == n)
						break;
				}
				if (!fz_is_empty_rect(fz_intersect_rect(list->tables[i].data.bounds, list->tables[k].data.bounds)))
				{
					if (list->tables[i].data.score > list->tables[k].data.score)
					{
						/* We intersect another table that scores better than us. Remove us. */
						break;
					}
					else if (list->tables[i].data.score == list->tables[k].data.score && i < k)
					{
						/* We intersect another table that scores the same as us, and it's bigger. Remove us. */
						break;
					}
				}
			}
			if (k != n)
			{
				fz_drop_potential_table(ctx, &list->tables[i]);
				continue;
			}
			if (i != j)
				list->tables[j] = list->tables[i];
			j++;
		}
		list->len = j;

#ifdef DEBUG_TABLE_SCORES
		list_tables(ctx, list, "final tables");
#endif

		/* Cull the tables that score too high. */
		n = list->len;
		for (j = 0, i = 0; i < n; i++)
		{
			if (list->tables[i].data.score > limit)
			{
				fz_drop_potential_table(ctx, &list->tables[i]);
				continue;
			}
			if (i != j)
				list->tables[j] = list->tables[i];
			j++;
		}
		list->len = j;

#ifdef DEBUG_TABLE_SCORES
		list_tables(ctx, list, "final final tables");
#endif

		/* Transcribe the remaining tables */
		n = list->len;
		for (i = 0; i < n; i++)
		{
#ifndef NDEBUG
			fz_verify_stext_page(ctx, page, "pre transcribe");
#endif
			last = transcribe_table(ctx, &list->tables[i].data, page, NULL);
		}
#ifndef NDEBUG
		fz_verify_stext_page(ctx, page, "post transcribe");
#endif
	}
	fz_always(ctx)
		fz_drop_potential_table_list(ctx, list);
	fz_catch(ctx)
		fz_rethrow(ctx);

	return last;
}

void
fz_table_hunt(fz_context *ctx, fz_stext_page *page)
{
	fz_table_hunt_within_bounds(ctx, page, fz_infinite_rect);
}

static void
push_trimmed_segment(fz_context *ctx, fz_potential_table_list *list, fz_stext_page *page, fz_stext_struct *parent, fz_rect bounds)
{
	fz_stext_page_block_iterator iter;
	int reduced = 0;

	/* Skip over a prefix of simple lines */
	for (iter = fz_stext_page_block_iterator_begin_from_dfs(page, parent ? parent->first_block : page->first_block, parent);
		!fz_stext_page_block_iterator_eod_dfs(iter);
		iter = fz_stext_page_block_iterator_next_dfs(iter))
	{
		fz_stext_line *line;
		while (1)
		{
			/* While we have a block of text at the top of our bounds. */
			if (iter.block->type != FZ_STEXT_BLOCK_TEXT)
				break;
			if (iter.block->bbox.y0 != bounds.y0)
				break;
			/* Skip over single span lines at the top of the bounds */
			line = iter.block->u.t.first_line;
			while (line)
			{
				if (line->bbox.y0 != bounds.y0)
					goto prefix_ended;
				if (line->next == NULL)
				{
					/* Skip this line */
					bounds.y0 = line->bbox.y1;
					reduced = 1;
				}
				else if (line->next->bbox.y0 >= line->bbox.y1)
				{
					/* Skip this line. */
					bounds.y0 = line->next->bbox.y0;
					reduced = 1;
				}
				else
					goto prefix_ended;
				line = line->next;
			}
			iter = fz_stext_page_block_iterator_next_dfs(iter);
			if (fz_stext_page_block_iterator_eod_dfs(iter))
				goto prefix_ended;
			bounds.y0 = iter.block->bbox.y0;
		}
	}
prefix_ended:

	/* Now do all that again, but for the suffix. */
	/* Skip over a suffix of simple lines */
	for (iter = fz_stext_page_block_iterator_begin_from_rdfs(page, parent ? parent->last_block : page->last_block, parent);
		!fz_stext_page_block_iterator_eod_rdfs(iter);
		iter = fz_stext_page_block_iterator_next_rdfs(iter))
	{
		fz_stext_line *line;
		while (1)
		{
			/* While we have a block of text at the bottom of our bounds. */
			if (iter.block->type != FZ_STEXT_BLOCK_TEXT)
				break;
			if (iter.block->bbox.y1 != bounds.y1)
				break;
			/* Skip over single span lines at the bottom of the bounds */
			line = iter.block->u.t.last_line;
			while (line)
			{
				if (line->bbox.y1 != bounds.y1)
					goto suffix_ended;
				if (line->prev == NULL)
				{
					/* Skip this line */
					bounds.y1 = line->bbox.y0;
					reduced = 1;
				}
				else if (line->prev->bbox.y1 <= line->bbox.y0)
				{
					/* Skip this line. */
					bounds.y1 = line->prev->bbox.y1;
					reduced = 1;
				}
				else
				{
					goto suffix_ended;
				}
				line = line->prev;
			}
			iter = fz_stext_page_block_iterator_next_rdfs(iter);
			if (fz_stext_page_block_iterator_eod_rdfs(iter))
				goto suffix_ended;
			bounds.y1 = iter.block->bbox.y1;
		}
	}
suffix_ended:
	if (reduced)
		fz_push_potential_table(ctx, list, bounds, 0);
}

static void
push_segment_areas(fz_context *ctx, fz_potential_table_list *list, fz_stext_page *page, fz_stext_struct *parent, fz_rect bounds, fz_rect parentr)
{
	fz_stext_block *first_block = parent ? parent->first_block : page->first_block;
	fz_stext_block *block;

	for (block = first_block; block != NULL; block = block->next)
	{
		fz_rect r;
		if (block->type != FZ_STEXT_BLOCK_STRUCT || block->u.s.down == NULL)
			continue;
		r = fz_intersect_rect(block->bbox, bounds);
		if (fz_is_empty_rect(r))
			continue;
		push_segment_areas(ctx, list, page, block->u.s.down, bounds, block->bbox);
		if (block->u.s.down->standard == FZ_STRUCTURE_DIV && !strcmp(block->u.s.down->raw, "Split") && fz_contains_rect(bounds, block->bbox))
		{
			assert(fz_contains_rect(parentr, block->bbox));
			/* The whole segment area is a possible table. */
			fz_push_potential_table(ctx, list, block->bbox, 0);

			/* Now consider trimming some text lines off the top and bottom (for title and caption lines) */
			push_trimmed_segment(ctx, list, page, block->u.s.down, r);
		}
	}
}

void
fz_table_hunt_within_bounds(fz_context *ctx, fz_stext_page *page, fz_rect bounds)
{
	fz_potential_table_list *list;
	fz_flotilla *flot = NULL;

	if (page == NULL)
		return;

	list = fz_new_potential_table_list(ctx);

	fz_var(flot);

	fz_try(ctx)
	{
		int i, n;

		/* First, find the different vector rafts on the page. */
		flot = fz_new_flotilla_from_stext_page_vectors(ctx, page);
		n = fz_flotilla_size(ctx, flot);

		/* Now push an entry for every raft that is contained within bounds. */
		for (i = 0; i < n; i++)
		{
			fz_rect r = fz_flotilla_raft_area(ctx, flot, i);

			if (!fz_contains_rect(bounds, r))
				continue;

			fz_push_potential_table(ctx, list, r, 1);
		}

		/* Now push potential table areas by walking for segments. */
		push_segment_areas(ctx, list, page, NULL, bounds, fz_infinite_rect);
	}
	fz_always(ctx)
		fz_drop_flotilla(ctx, flot);
	fz_catch(ctx)
	{
		fz_drop_potential_table_list(ctx, list);
		fz_rethrow(ctx);
	}

	hunt_potential_tables(ctx, page, list, 0.3f/* Nasty heuristic constant! */);
}

fz_stext_block *
fz_find_table_within_bounds(fz_context *ctx, fz_stext_page *page, fz_rect bounds)
{
	fz_potential_table_list *list;

	if (page == NULL)
		return NULL;

	/* Make a list, even though we're only going to push one entry in. */
	list = fz_new_potential_table_list(ctx);

	fz_try(ctx)
		fz_push_potential_table(ctx, list, bounds, 0);
	fz_catch(ctx)
	{
		fz_drop_potential_table_list(ctx, list);
		fz_rethrow(ctx);
	}

	return hunt_potential_tables(ctx, page, list, 9999999.0f);
}

fz_stext_grid_positions *
fz_clone_stext_grid_positions(fz_context *ctx, fz_stext_grid_positions *src)
{
	fz_stext_grid_positions *dst;

	if (src == NULL)
		return NULL;

	dst = fz_malloc_flexible(ctx, fz_stext_grid_positions, list, src->len);
	memcpy(dst, src, fz_sizeof_flexible(fz_stext_grid_positions, list, src->len));

	return dst;
}

fz_stext_block *
fz_find_table_within_grid(fz_context *ctx, fz_stext_page *page, fz_stext_grid_positions *xpos, fz_stext_grid_positions *ypos, float limit)
{
	fz_potential_table_list *list;
	fz_rect bounds;
	int w, h;

	if (page == NULL || xpos == NULL || ypos == NULL || xpos->len < 2 || ypos->len < 2)
		return NULL;

	/* Make a list, even though we're only going to push one entry in. */
	list = fz_new_potential_table_list(ctx);

	w = xpos->len;
	h = ypos->len;
	bounds.x0 = xpos->list[0].min;
	bounds.x1 = xpos->list[w-1].max;
	bounds.y0 = ypos->list[0].min;
	bounds.y1 = ypos->list[h-1].max;
	fz_try(ctx)
	{
		fz_push_potential_table(ctx, list, bounds, 0);
		list->tables[0].data.xpos = fz_clone_stext_grid_positions(ctx, xpos);
		list->tables[0].data.ypos = fz_clone_stext_grid_positions(ctx, ypos);
	}
	fz_catch(ctx)
	{
		fz_drop_potential_table_list(ctx, list);
		fz_rethrow(ctx);
	}

	return hunt_potential_tables(ctx, page, list, limit);
}
