Commit: c53b77c930cba0138049a5ea78f6ee961c5f9e65 - pukiwiki-fork (git)

Commit MetaInfo

• Revision: c53b77c930cba0138049a5ea78f6ee961c5f9e65 (tree)
• Time: 2018-02-27 00:10:34
• Author: umorigu <umorigu@gmai...>
• Commiter: umorigu

Log Message

BugTrack/560 tracker_list speeding up with cache

* Implement 2-layer caching

• Converted HTML cache
  
• Contents cache for list pages
  

* Get only updated/deleted page contents
* Detect page updates by file or page timestamps
* Refer cache/recent.dat file to detect updated pages
* Refer RecentDeleted page to detect deleted pages
* Handle page links properly on tracker_list table
* Save contents cache on cache/(TrackerBase).tracker

How to invalidate cache:

Execute one of following actions.

* Change tracker config file
* Change tracker list file
* Delete cache/*.tracker file manually

Change Summary

• modified: plugin/tracker.inc.php
• modified: plugin/tracker_list.inc.php

Incremental Difference

--- a/plugin/tracker.inc.php
+++ b/plugin/tracker.inc.php
@@ -1,7 +1,7 @@
 <?php
 // PukiWiki - Yet another WikiWikiWeb clone
 // tracker.inc.php
-// Copyright 2003-2017 PukiWiki Development Team
+// Copyright 2003-2018 PukiWiki Development Team
 // License: GPL v2 or (at your option) any later version
 //
 // Issue tracker plugin (See Also bugtrack plugin)
@@ -15,6 +15,9 @@ define('TRACKER_LIST_EXCLUDE_PATTERN','#^SubMenu$|/#');
 // 項目の取り出しに失敗したページを一覧に表示する
 define('TRACKER_LIST_SHOW_ERROR_PAGE',TRUE);
 
+// Use cache
+define('TRACKER_LIST_USE_CACHE', TRUE);
+
 function plugin_tracker_convert()
 {
 	global $vars;
@@ -684,13 +687,13 @@ function plugin_tracker_list_action()
 }
 function plugin_tracker_getlist($page,$refer,$config_name,$list,$order='',$limit=NULL)
 {
-	$config = new Config('plugin/tracker/'.$config_name);
+	global $whatsdeleted;
 
+	$config = new Config('plugin/tracker/'.$config_name);
 	if (!$config->read())
 	{
 		return "<p>config file '".htmlsc($config_name)."' is not exist.</p>";
 	}
-
 	$config->config_name = $config_name;
 
 	if (!is_page($config->page.'/'.$list))
@@ -698,9 +701,79 @@ function plugin_tracker_getlist($page,$refer,$config_name,$list,$order='',$limit
 		return "<p>config file '".make_pagelink($config->page.'/'.$list)."' not found.</p>";
 	}
 
-	$list = new Tracker_list($page,$refer,$config,$list);
-	$list->sort($order);
-	return $list->toString($limit);
+	$cache_enabled = defined('TRACKER_LIST_USE_CACHE') && TRACKER_LIST_USE_CACHE &&
+		defined('JSON_UNESCAPED_UNICODE') && defined('PKWK_UTF8_ENABLE');
+	$cache_filepath = CACHE_DIR . encode($page) . '.tracker';
+	$cachedata = null;
+	$cache_format_version = 1;
+	if ($cache_enabled && file_exists($cache_filepath)) {
+		$config_filetime = get_filetime($config->page);
+		$config_list_filetime = get_filetime($config->page.'/'. $list);
+		$json_cached = pkwk_file_get_contents($cache_filepath);
+		if ($json_cached) {
+			$wrapdata = json_decode($json_cached, true);
+			if (is_array($wrapdata)) {
+				if (isset($wrapdata['version'], $wrapdata['html'], $wrapdata['refreshed_at'])) {
+					$cache_time_prev = $wrapdata['refreshed_at'];
+					if ($cache_format_version === $wrapdata['version']) {
+						if ($config_filetime === $wrapdata['config_updated_at'] &&
+							$config_list_filetime === $wrapdata['config_list_updated_at']) {
+							$cachedata = $wrapdata;
+						} else {
+							// (Ignore) delete file
+							unlink($cache_filepath);
+						}
+					}
+				}
+			}
+		}
+	}
+	// Check recent.dat timestamp
+	$recent_dat_filemtime = filemtime(CACHE_DIR . PKWK_MAXSHOW_CACHE);
+	// Check RecentDeleted timestamp
+	$recent_deleted_filetime = get_filetime($whatsdeleted);
+	if (is_null($cachedata)) {
+		$cachedata = array();
+	} else {
+		if ($recent_dat_filemtile !== false) {
+			if ($recent_dat_filemtime === $cachedata['recent_dat_filemtime'] &&
+				$recent_deleted_filetime === $cachedata['recent_deleted_filetime'] &&
+				$order === $cachedata['order']) {
+				// recent.dat is unchanged
+				// RecentDeleted is unchanged
+				// order is unchanged
+				return $cachedata['html'];
+			}
+		}
+	}
+	$cache_holder = $cachedata;
+	$tracker_list = new Tracker_list($page,$refer,$config,$list,$cache_holder);
+	if ($order === $cache_holder['order'] &&
+		empty($tracker_list->newly_deleted_pages) &&
+		empty($tracker_list->newly_updated_pages) &&
+		!$tracker_list->link_update_required) {
+		$result = $cache_holder['html'];
+	} else {
+		$tracker_list->sort($order);
+		$result = $tracker_list->toString($limit);
+	}
+	if ($cache_enabled) {
+		$refreshed_at = time();
+		$json = array(
+			'refreshed_at' => $refreshed_at,
+			'rows' => $tracker_list->rows,
+			'html' => $result,
+			'order' => $order,
+			'config_updated_at' => $config_filetime,
+			'config_list_updated_at' => $config_list_filetime,
+			'recent_dat_filemtime' => $recent_dat_filemtime,
+			'recent_deleted_filetime' => $recent_deleted_filetime,
+			'link_pages' => $tracker_list->link_pages,
+			'version' => $cache_format_version);
+		$cache_body = json_encode($json, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
+		file_put_contents($cache_filepath, $cache_body, LOCK_EX);
+	}
+	return $result;
 }
 
 // 一覧クラス
@@ -715,13 +788,16 @@ class Tracker_list
 	var $rows;
 	var $order;
 	var $sort_keys;
+	var $newly_deleted_pages = array();
+	var $newly_updated_pages = array();
 
-	function Tracker_list($page,$refer,&$config,$list)
+	function Tracker_list($page,$refer,&$config,$list,&$cache_holder)
 	{
-		$this->__construct($page, $refer, $config, $list);
+		$this->__construct($page, $refer, $config, $list, $cache_holder);
 	}
-	function __construct($page,$refer,&$config,$list)
+	function __construct($page,$refer,&$config,$list,&$cache_holder)
 	{
+		global $whatsdeleted;
 		$this->page = $page;
 		$this->config = &$config;
 		$this->list = $list;
@@ -746,22 +822,188 @@ class Tracker_list
 				$this->pattern .= '(.*?)';
 			}
 		}
-		// ページの列挙と取り込み
-		$this->rows = array();
-		$pattern = "$page/";
-		$pattern_len = strlen($pattern);
-		foreach (get_existpages() as $_page)
-		{
-			if (strpos($_page,$pattern) === 0)
+		if (empty($cache_holder)) {
+			// List pages and get contents (non-cache behavior)
+			$this->rows = array();
+			$pattern = "$page/";
+			$pattern_len = strlen($pattern);
+			foreach (get_existpages() as $_page)
 			{
-				$name = substr($_page,$pattern_len);
-				if (preg_match(TRACKER_LIST_EXCLUDE_PATTERN,$name))
+				if (substr($_page, 0, $pattern_len) === $pattern)
 				{
-					continue;
+					$name = substr($_page,$pattern_len);
+					if (preg_match(TRACKER_LIST_EXCLUDE_PATTERN,$name))
+					{
+						continue;
+					}
+					$this->add($_page,$name);
+				}
+			}
+			$this->link_pages = $this->get_filetimes($this->get_all_links());
+		} else {
+			// Cache-available behavior
+			// Check RecentDeleted timestamp
+			$cached_rows = $this->decode_cached_rows($cache_holder['rows']);
+			$updated_linked_pages = array();
+			$newly_deleted_pages = array();
+			$pattern = "$page/";
+			$pattern_len = strlen($pattern);
+			$recent_deleted_filetime = get_filetime($whatsdeleted);
+			$deleted_page_list = array();
+			if ($recent_deleted_filetime !== $cache_holder['recent_deleted_filetime']) {
+				//pkwk_log(print_r());
+				foreach (plugin_tracker_get_source($whatsdeleted) as $line) {
+					$m = null;
+					if (preg_match('#\[\[([^\]]+)\]\]#', $line, $m)) {
+						$_page = $m[1];
+						if (is_pagename($_page)) {
+							$deleted_page_list[] = $m[1];
+						}
+					}
+				}
+				foreach ($deleted_page_list as $_page) {
+					if (substr($_page, 0, $pattern_len) === $pattern) {
+						$name = substr($_page, $pattern_len);
+						if (!is_page($_page) && isset($cached_rows[$name]) &&
+							!preg_match(TRACKER_LIST_EXCLUDE_PATTERN, $name)) {
+							// This page was just deleted
+							array_push($newly_deleted_pages, $_page);
+							unset($cached_rows[$name]);
+						}
+					}
 				}
-				$this->add($_page,$name);
 			}
+			$this->newly_deleted_pages = $newly_deleted_pages;
+			$updated_pages = array();
+			$this->rows = $cached_rows;
+			// Check recent.dat timestamp
+			$recent_dat_filemtime = filemtime(CACHE_DIR . PKWK_MAXSHOW_CACHE);
+			$updated_page_list = array();
+			if ($recent_dat_filemtime !== $cache_holder['recent_dat_filemtime']) {
+				// recent.dat was updated. Search which page was updated.
+				$target_pages = array();
+				// Active page file time (1 hour before timestamp of recent.dat)
+				$target_filetime = $cache_holder['recent_dat_filemtime'] - LOCALZONE - 60 * 60;
+				foreach (get_recent_files() as $_page=>$time) {
+					if ($time <= $target_filetime) {
+						// Older updated pages
+						break;
+					}
+					$updated_page_list[$_page] = $time;
+					$name = substr($_page, $pattern_len);
+					if (substr($_page, 0, $pattern_len) === $pattern) {
+						$name = substr($_page, $pattern_len);
+						if (preg_match(TRACKER_LIST_EXCLUDE_PATTERN, $name)) {
+							continue;
+						}
+						// Tracker target page
+						if (isset($this->rows[$name])) {
+							// Existing page
+							$row = $this->rows[$name];
+							if ($row['_update'] === get_filetime($_page)) {
+								// Same as cache
+								continue;
+							} else {
+								// Found updated page
+								$updated_pages[] = $_page;
+								unset($this->rows[$name]);
+								$this->add($_page, $name);
+							}
+						} else {
+							// Add new page
+							$updated_pages[] = $_page;
+							$this->add($_page, $name);
+						}
+					}
+				}
+			}
+			$this->newly_updated_pages = $updated_pages;
+			$new_link_names = $this->get_all_links();
+			$old_link_map = array();
+			foreach ($cache_holder['link_pages'] as $link_page) {
+				$old_link_map[$link_page['page']] = $link_page['filetime'];
+			}
+			$new_link_map = $old_link_map;
+			$link_update_required = false;
+			foreach ($deleted_page_list as $_page) {
+				if (in_array($_page, $new_link_names)) {
+					if (isset($old_link_map[$_page])) {
+						// This link keeps existing
+						if (!is_page($_page)) {
+							// OK. Confirmed the page doesn't exist
+							if ($old_link_map[$_page] === 0) {
+								// Do nothing (From no-page to no-page)
+							} else {
+								// This page was just deleted
+								$new_link_map[$_page] = get_filetime($_page);
+								$link_update_required = true;
+							}
+						}
+					} else {
+						// This link was just added
+						$new_link_map[$_page] = get_filetime($_page);
+						$link_update_required = true;
+					}
+				}
+			}
+			foreach ($updated_page_list as $_page=>$time) {
+				if (in_array($_page, $new_link_names)) {
+					if (isset($old_link_map[$_page])) {
+						// This link keeps existing
+						if (is_page($_page)) {
+							// OK. Confirmed the page now exists
+							if ($old_link_map[$_page] === 0) {
+								// This page was just added
+								$new_link_map[$_page] = get_filetime($_page);
+								$link_update_required = true;
+							} else {
+								// Do nothing (existing-page to existing-page)
+							}
+						}
+					} else {
+						// This link was just added
+						$new_link_map[$_page] = get_filetime($_page);
+						$link_update_required = true;
+					}
+				}
+			}
+			$new_link_pages = array();
+			foreach ($new_link_map as $_page => $time) {
+				$new_link_pages[] = array(
+					'page' => $_page,
+					'filetime' => $time,
+				);
+			}
+			$this->link_pages = $new_link_pages;
+			$this->link_update_required = $link_update_required;
+		}
+	}
+	function decode_cached_rows($decoded_rows)
+	{
+		$ar = array();
+		foreach ($decoded_rows as $row) {
+			$ar[$row['_real']] = $row;
+		}
+		return $ar;
+	}
+	function get_all_links() {
+		$ar = array();
+		foreach ($this->rows as $row) {
+			foreach ($row['_links'] as $link) {
+				$ar[$link] = 0;
+			}
+		}
+		return array_keys($ar);
+	}
+	function get_filetimes($pages) {
+		$filetimes = array();
+		foreach ($pages as $page) {
+			$filetimes[] = array(
+				'page' => $page,
+				'filetime' => get_filetime($page),
+			);
 		}
+		return $filetimes;
 	}
 	function add($page,$name)
 	{
@@ -786,22 +1028,38 @@ class Tracker_list
 		}
 		$source = join('',preg_replace('/^(\*{1,3}.*)\[#[A-Za-z][\w-]+\](.*)$/','$1$2',$source));
 
-		// デフォルト値
-		$this->rows[$name] = array(
+		// Default value
+		$page_filetime = get_filetime($page);
+		$row = array(
 			'_page'  => "[[$page]]",
 			'_refer' => $this->page,
 			'_real'  => $name,
-			'_update'=> get_filetime($page),
-			'_past'  => get_filetime($page)
+			'_update'=> $page_filetime,
+			'_past'  => $page_filetime,
 		);
-		if ($this->rows[$name]['_match'] = preg_match("/{$this->pattern}/s",$source,$matches))
+		$links = array();
+		if ($row['_match'] = preg_match("/{$this->pattern}/s",$source,$matches))
 		{
 			array_shift($matches);
 			foreach ($this->pattern_fields as $key=>$field)
 			{
-				$this->rows[$name][$field] = trim($matches[$key]);
+				$row[$field] = trim($matches[$key]);
+				if ($field === '_refer') {
+					continue;
+				}
+				$lmatch = null;
+				if (preg_match('/\[\[([^\]\]]+)\]/', $row[$field], $lmatch)) {
+					$link = $lmatch[1];
+					if (is_pagename($link) && $link !== $this->page && $link !== $page) {
+						if (!in_array($link, $links)) {
+							$links[] = $link;
+						}
+					}
+				}
 			}
 		}
+		$row['_links'] = $links;
+		$this->rows[$name] = $row;
 	}
 	function compare($a, $b)
 	{

--- a/plugin/tracker_list.inc.php
+++ b/plugin/tracker_list.inc.php
@@ -1,7 +1,7 @@
 <?php
 // PukiWiki - Yet another WikiWikiWeb clone
 // tracker_list.inc.php
-// Copyright 2003-2017 PukiWiki Development Team
+// Copyright 2003-2018 PukiWiki Development Team
 // License: GPL v2 or (at your option) any later version
 //
 // Issue tracker list plugin (a part of tracker plugin)