Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Removed non-METATILE code as it no longer seems to work #366

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ Apache file serving mechanisms to provide:
5) Tile expiry. It estimates when the tile is next likely to be rendered and adds the appropriate HTTP cache expiry headers. This is a configurable heuristic.

To avoid problems with directories becoming too large and to avoid
too many tiny files. They store the rendered tiles in "meta tiles" in a
too many tiny files. They store the rendered tiles in "metatiles" in a
special hashed directory structure. These combine 8x8 actual tiles into a
single metatile file. This is a more efficient use of disk space and inodes.

Expand Down
15 changes: 1 addition & 14 deletions includes/render_config.h
Original file line number Diff line number Diff line change
Expand Up @@ -102,28 +102,15 @@
// default for number of rendering threads
#define NUM_THREADS (4)

// Use this to enable meta-tiles which will render NxN tiles at once
// Metatiles will render NxN tiles at once
// Note: This should be a power of 2 (2, 4, 8, 16 ...)
#define METATILE (8)
//#undef METATILE

//Fallback to standard tiles if meta tile doesn't exist
//Legacy - not needed on new installs
//#undef METATILEFALLBACK

// Metatiles are much larger in size so we don't need big queues to handle large areas
#ifdef METATILE
#define QUEUE_MAX (64)
#define REQ_LIMIT (256)
#define DIRTY_LIMIT (8000)

#else
#define QUEUE_MAX (1024)
#define REQ_LIMIT (512)
#define DIRTY_LIMIT (10000)
#define HASHIDX_SIZE 22123
#endif

// Penalty for client making an invalid request (in seconds)
#define CLIENT_PENALTY (3)

Expand Down
7 changes: 2 additions & 5 deletions includes/store_file_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -33,18 +33,15 @@ extern "C" {
int mkdirp(const char *path);

/* File path hashing. Used by both mod_tile and render daemon
* The two must both agree on the file layout for meta-tiling
* The two must both agree on the file layout for metatiling
* to work
*/
void xyz_to_path(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z);

int path_to_xyz(const char *tilepath, const char *path, char *xmlconfig, int *px, int *py, int *pz);

#ifdef METATILE
/* New meta-tile storage functions */
/* Returns the path to the meta-tile and the offset within the meta-tile */
/* Returns the path to the metatile and the offset within the metatile */
int xyz_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z);
#endif

#ifdef __cplusplus
}
Expand Down
61 changes: 1 addition & 60 deletions src/gen_tile.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -72,11 +72,7 @@ using namespace mapnik;
#define RAD_TO_DEG (180/M_PI)
#endif

#ifdef METATILE
#define RENDER_SIZE (256 * (METATILE + 1))
#else
#define RENDER_SIZE (512)
#endif

extern int foreground;

Expand Down Expand Up @@ -245,7 +241,6 @@ static int check_xyz(int x, int y, int z, struct xmlmapconfig * map)
return !oob;
}

#ifdef METATILE
mapnik::box2d<double> tile2prjbounds(struct projectionconfig * prj, int x, int y, int z)
{

Expand Down Expand Up @@ -299,7 +294,7 @@ static enum protoCmd render(struct xmlmapconfig * map, int x, int y, int z, char
return cmdNotDone;
}

// Split the meta tile into an NxN grid of tiles
// Split the metatile into an NxN grid of tiles
unsigned int xx, yy;

for (yy = 0; yy < render_size_ty; yy++) {
Expand All @@ -312,52 +307,6 @@ static enum protoCmd render(struct xmlmapconfig * map, int x, int y, int z, char

return cmdDone; // OK
}
#else //METATILE
static enum protoCmd render(Map &m, const char *tile_dir, char *xmlname, projection &prj, int x, int y, int z, char* outputFormat)
{
char filename[PATH_MAX];
char tmp[PATH_MAX];
double p0x = x * 256.0;
double p0y = (y + 1) * 256.0;
double p1x = (x + 1) * 256.0;
double p1y = y * 256.0;

tiling.fromPixelToLL(p0x, p0y, z);
tiling.fromPixelToLL(p1x, p1y, z);

prj.forward(p0x, p0y);
prj.forward(p1x, p1y);

mapnik::box2d<double> bbox(p0x, p0y, p1x, p1y);
bbox.width(bbox.width() * 2);
bbox.height(bbox.height() * 2);
m.zoomToBox(bbox);

mapnik::image_32 buf(RENDER_SIZE, RENDER_SIZE);
mapnik::agg_renderer<mapnik::image_32> ren(m, buf);
ren.apply();

xyz_to_path(filename, sizeof(filename), tile_dir, xmlname, x, y, z);

if (mkdirp(filename)) {
return cmdNotDone;
}

snprintf(tmp, sizeof(tmp), "%s.tmp", filename);

mapnik::image_view<mapnik::image_data_32> vw(128, 128, 256, 256, buf.data());
g_logger(G_LOG_LEVEL_DEBUG, "Render %i %i %i %s", z, x, y, filename)
mapnik::save_to_file(vw, tmp, outputFormat);

if (rename(tmp, filename)) {
perror(tmp);
return cmdNotDone;
}

return cmdDone; // OK
}
#endif //METATILE


void render_init(const char *plugins_dir, const char* font_dir, int font_dir_recurse)
{
Expand Down Expand Up @@ -451,7 +400,6 @@ void *render_thread(void * arg)

if (item) {
struct protocol *req = &item->req;
#ifdef METATILE
// At very low zoom the whole world may be smaller than METATILE
unsigned int size = MIN(METATILE, 1 << req->z);

Expand Down Expand Up @@ -503,13 +451,6 @@ void *render_thread(void * arg)
request_exit();
}
}

#else //METATILE
ret = render(maps[i].map, maps[i].tile_dir, req->xmlname, maps[i].prj, req->x, req->y, req->z, maps[i].output_format);
#ifdef HTCP_EXPIRE_CACHE
cache_expire(maps[i].htcpsock, maps[i].host, maps[i].xmluri, req->x, req->y, req->z);
#endif
#endif //METATILE
} else {
g_logger(G_LOG_LEVEL_WARNING, "Received request for map layer %s is outside of acceptable bounds z(%i), x(%i), y(%i)",
req->xmlname, req->z, req->x, req->y);
Expand Down
2 changes: 1 addition & 1 deletion src/metatile.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ const std::string metaTile::get(int x, int y)
return tile[x][y];
}

// Returns the offset within the meta-tile index table
// Returns the offset within the metatile index table
int metaTile::xyz_to_meta_offset(int x, int y, int z)
{
unsigned char mask = METATILE - 1;
Expand Down
23 changes: 6 additions & 17 deletions src/render_expired.c
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ const char * tile_dir_default = RENDERD_TILE_DIR;

// macros handling our tile marking arrays (these are essentially bit arrays
// that have one bit for each tile on the repsective zoom level; since we only
// need them for meta tile levels, even if someone were to render level 20,
// need them for metatile levels, even if someone were to render level 20,
// we'd still only use 4^17 bits = 2 GB RAM (plus a little for the lower zoom
// levels) - this saves us the hassle of working with a tree structure.

Expand All @@ -52,16 +52,6 @@ const char * tile_dir_default = RENDERD_TILE_DIR;
#define SET_TILE_REQUESTED(z,x,y) \
tile_requested[z][((x)*twopow[z]+(y))/(8*sizeof(int))] |= (0x01 << (((x)*twopow[z]+(y))%(8*sizeof(int))));


#ifndef METATILE
#warning("render_expired not implemented for non-metatile mode. Feel free to submit fix")
int main(int argc, char **argv)
{
fprintf(stderr, "render_expired not implemented for non-metatile mode. Feel free to submit fix!\n");
return -1;
}
#else

// tile marking arrays
unsigned int **tile_requested;

Expand Down Expand Up @@ -112,7 +102,7 @@ int main(int argc, char **argv)
char name[PATH_MAX];

// excess_zoomlevels is how many zoom levels at the large end
// we can ignore because their tiles will share one meta tile.
// we can ignore because their tiles will share one metatile.
// with the default METATILE==8 this is 3.
int excess_zoomlevels = 0;
int mt = METATILE;
Expand Down Expand Up @@ -399,7 +389,7 @@ int main(int argc, char **argv)
{
gettimeofday(&end, NULL);
printf("\n");
printf("Meta tiles rendered: ");
printf("Metatiles rendered: ");
display_rate(start, end, num_render);
printf("Total tiles rendered: ");
display_rate(start, end, num_render * METATILE * METATILE);
Expand Down Expand Up @@ -443,16 +433,15 @@ int main(int argc, char **argv)

gettimeofday(&end, NULL);
printf("\nTotal for all tiles rendered\n");
printf("Meta tiles rendered: ");
printf("Metatiles rendered: ");
Dismissed Show dismissed Hide dismissed
display_rate(start, end, num_render);
printf("Total tiles rendered: ");
display_rate(start, end, num_render * METATILE * METATILE);
printf("Total tiles in input: %d\n", num_read);
printf("Total tiles expanded from input: %d\n", num_all);
printf("Total meta tiles deleted: %d\n", num_unlink);
printf("Total meta tiles touched: %d\n", num_touch);
printf("Total metatiles deleted: %d\n", num_unlink);
Dismissed Show dismissed Hide dismissed
printf("Total metatiles touched: %d\n", num_touch);
Dismissed Show dismissed Hide dismissed
printf("Total tiles ignored (not on disk): %d\n", num_ignore);

return 0;
}
#endif
14 changes: 2 additions & 12 deletions src/render_list.c
Original file line number Diff line number Diff line change
Expand Up @@ -44,15 +44,6 @@

const char * tile_dir_default = RENDERD_TILE_DIR;

#ifndef METATILE
#warning("render_list not implemented for non-metatile mode. Feel free to submit fix")
int main(int argc, char **argv)
{
fprintf(stderr, "render_list not implemented for non-metatile mode. Feel free to submit fix!\n");
return -1;
}
#else

static int minZoom = 0;
static int maxZoom = MAX_ZOOM;
static int verbose = 0;
Expand Down Expand Up @@ -372,7 +363,7 @@ int main(int argc, char **argv)
if (!(num_render % 10)) {
gettimeofday(&end, NULL);
printf("\n");
printf("Meta tiles rendered: ");
printf("Metatiles rendered: ");
Dismissed Show dismissed Hide dismissed
display_rate(start, end, num_render);
printf("Total tiles rendered: ");
display_rate(start, end, (num_render) * METATILE * METATILE);
Expand Down Expand Up @@ -406,7 +397,7 @@ int main(int argc, char **argv)
printf("\n*****************************************************\n");
printf("*****************************************************\n");
printf("Total for all tiles rendered\n");
printf("Meta tiles rendered: ");
printf("Metatiles rendered: ");
Dismissed Show dismissed Hide dismissed
display_rate(start, end, num_render);
printf("Total tiles rendered: ");
display_rate(start, end, num_render * METATILE * METATILE);
Expand All @@ -416,4 +407,3 @@ int main(int argc, char **argv)

return 0;
}
#endif
12 changes: 1 addition & 11 deletions src/render_old.c
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,6 @@

const char * tile_dir_default = RENDERD_TILE_DIR;

#ifndef METATILE
#warning("render_old not implemented for non-metatile mode. Feel free to submit fix")
int main(int argc, char **argv)
{
fprintf(stderr, "render_old not implemented for non-metatile mode. Feel free to submit fix!\n");
return -1;
}
#else

#define INILINE_MAX 256
static int minZoom = 0;
static int maxZoom = MAX_ZOOM;
Expand Down Expand Up @@ -400,7 +391,7 @@ int main(int argc, char **argv)

gettimeofday(&end, NULL);
printf("\nTotal for all tiles rendered\n");
printf("Meta tiles rendered: ");
printf("Metatiles rendered: ");
Dismissed Show dismissed Hide dismissed
display_rate(start, end, num_render);
printf("Total tiles rendered: ");
display_rate(start, end, num_render * METATILE * METATILE);
Expand All @@ -409,4 +400,3 @@ int main(int argc, char **argv)

return 0;
}
#endif
10 changes: 0 additions & 10 deletions src/render_speedtest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,6 @@
#define DEG_TO_RAD (M_PI/180)
#define RAD_TO_DEG (180/M_PI)

#ifndef METATILE
#warning("Speed test not implemented for non-metatile mode. Feel free to submit fix")
int main(int argc, char **argv)
{
fprintf(stderr, "Speed test not implemented for non-metatile mode. Feel free to submit fix!\n");
return -1;
}
#else

#if 1
static double boundx0 = -0.5;
static double boundy0 = 51.25;
Expand Down Expand Up @@ -295,4 +286,3 @@ int main(int argc, char **argv)

return ret;
}
#endif
5 changes: 0 additions & 5 deletions src/renderd.c
Original file line number Diff line number Diff line change
Expand Up @@ -166,17 +166,12 @@ enum protoCmd rx_request(struct protocol *req, int fd)
item->duplicates = NULL;
item->fd = (req->cmd == cmdDirty) ? FD_INVALID : fd;

#ifdef METATILE
/* Round down request co-ordinates to the neareast N (should be a power of 2)
* Note: request path is no longer consistent but this will be recalculated
* when the metatile is being rendered.
*/
item->mx = item->req.x & ~(METATILE - 1);
item->my = item->req.y & ~(METATILE - 1);
#else
item->mx = item->req.x;
item->my = item->req.y;
#endif

return request_queue_add_request(render_request_queue, item);
}
Expand Down
4 changes: 2 additions & 2 deletions src/store_file.c
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
* along with this program; If not, see http://www.gnu.org/licenses/.
*/

/* Meta-tile optimised file storage
/* Metatile optimised file storage
*
* Instead of storing each individual tile as a file,
* bundle the 8x8 meta tile into a special meta-file.
* bundle the 8x8 metatile into a special meta-file.
* This reduces the Inode usage and more efficient
* utilisation of disk space.
*/
Expand Down
12 changes: 5 additions & 7 deletions src/store_file_utils.c
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ int mkdirp(const char *path)


/* File path hashing. Used by both mod_tile and render daemon
* The two must both agree on the file layout for meta-tiling
* The two must both agree on the file layout for metatiling
* to work
*/

Expand Down Expand Up @@ -191,20 +191,19 @@ int path_to_xyz(const char *tilepath, const char *path, char *xmlconfig, int *px
#endif
}

#ifdef METATILE
// Returns the path to the meta-tile and the offset within the meta-tile
// Returns the path to the metatile and the offset within the metatile
int xyz_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z)
{
return xyzo_to_meta(path, len, tile_dir, xmlconfig, "", x, y, z);
}

// Returns the path to the meta-tile and the offset within the meta-tile
// Returns the path to the metatile and the offset within the metatile
int xyzo_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, const char *options, int x, int y, int z)
{
unsigned char i, hash[5], offset, mask;

// Each meta tile winds up in its own file, with several in each leaf directory
// the .meta tile name is beasd on the sub-tile at (0,0)
// Each metatile winds up in its own file, with several in each leaf directory
// the .meta file name is based on the sub-tile at (0,0)
mask = METATILE - 1;
offset = (x & mask) * METATILE + (y & mask);
x &= ~mask;
Expand Down Expand Up @@ -235,4 +234,3 @@ int xyzo_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlco
#endif
return offset;
}
#endif
Loading
Loading