package extensions;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.net.URLDecoder;
import java.net.Socket;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import dareka.Main;
import dareka.common.CloseUtil;
import dareka.common.DiskFreeSpace;
import dareka.common.Logger;
import dareka.common.NullOutputStream;
import dareka.common.Pair;
import dareka.common.json.*;
import dareka.extensions.Extension2;
import dareka.extensions.ExtensionManager;
import dareka.extensions.RequestFilter;
import dareka.processor.HttpRequestHeader;
import dareka.processor.HttpResponseHeader;
import dareka.processor.Resource;
import dareka.processor.URLResource;
import dareka.processor.StringResource;
import dareka.processor.TransferListener;
import dareka.processor.Processor;
import dareka.processor.impl.Cache;
import dareka.processor.impl.NicoCachingTitleRetriever;
import dareka.processor.impl.NicoIdInfoCache;
import dareka.processor.impl.ReEncodingInfo;
import dareka.processor.impl.VideoDescriptor;
import dareka.processor.impl.WatchVars;
import dareka.processor.util.GetThumbInfoUtil;

/*
 * とりあえず動画をキャッシュして後で見るよ
 */
public class nlMovieFetcher implements Extension2, Processor, RequestFilter,
		TransferListener, Runnable {
	
	public static final int REVISION = 170214;
	public static final String VER_STRING = "nlMovieFetcher_110411(v0.6) w/ " + REVISION + " o_O"; /* 121016 FIX + 20131111 mod + 170214 o_O */
	public static final String LOG_PREFIX = "fetch";
	public static final String PROP_DEBUG = "movieFetchDebug";
	
	private static final String[] PROCESSOR_SUPPORTED_METHODS = new String[] { "GET", "POST" };
	private static final Pattern PROCESSOR_SUPPORTED_PATTERN = Pattern.compile(
			"^http://www\\.nicovideo\\.jp/cache/fetch\\?" +
			"((?:(status|rm|cancel)=)?(?:\\w{2}\\d+|all)|(?:mylist=(?:\\d+|all))|list|json|storage|updatelow)(&.+|)$");
	
	private static final Map<String, String> fetchResults =
		Collections.synchronizedMap(new java.util.LinkedHashMap<String, String>());
	private static final Pair<File, Long> fetchResultsFileInfo =
			new Pair<File, Long>(new File("local/fetched.json"), 0L);
	private static volatile boolean fetchResultsUpdated;
	
	private static volatile String storageCache;
	private static final Pair<File, Long> storageFileInfo =
			new Pair<File, Long>(new File("local/fetched.storage"), 0L);
	private static volatile boolean storageUpdated;
	
	private static final String FETCH_URI = "fetch-uri";
	private static final Charset FILE_CHARSET = getCharset("movieFetchCharset");
	private static final StringResource resultOK = ExtUtil.responseText("OK");
	private static final StringResource resultNG = ExtUtil.responseText("NG");
	
	private static ExtUtil util;
	private static String user_session;
	
	private String fetchId;
	private HttpRequestHeader fetchRequestHeader;
	private ScheduledFuture<?> future;
	private volatile boolean canceled;
	private long contentLength, transferedLength, completedTime;
	private int premiumWait;
	
	private void debug2(String format, Object...args) {
		if(Boolean.getBoolean("movieFetchDebug2")) util.debug(format, args); }
	private void debugWithId(String mes){ util.debug("%s %s", this.fetchId, mes); }
	
	private static final String URL_DEFLIST_API = "http://www.nicovideo.jp/api/deflist/list";
	private static final String URL_MYLISTGROUP_API = "http://www.nicovideo.jp/api/mylistgroup/list";
	private static final String URL_MYLIST =  "http://www.nicovideo.jp/mylist/";
	private static final String URL_VIDEOARRAY_API = "http://api.ce.nicovideo.jp/nicoapi/v1/video.array?v=";
	private static final Pattern JSON_VIDEO_ID_PATTERN = Pattern.compile(
		"\"video_id\":\"([a-z]{2}\\d+)\"");
	private static final Pattern JSON_MYLIST_ID_PATTERN = Pattern.compile(
		"\"id\":\"(\\d+)\"");
	private static final Pattern XML_VIDEO_ID_PATTERN = Pattern.compile(
		"<video_info><video><id>([a-z]{2}\\d+)</id>");
	
	private static long multiple_access_timeout = 0;
	private static long last_access_time = System.currentTimeMillis();
	private static final long access_wait = 10 * 1000;
	private static final Semaphore access_sem = new Semaphore(1, true);
	
	private volatile boolean started;
	
	// 引数付きコンストラクタを定義した場合は引数無しコンストラクタも必要
	public nlMovieFetcher() {
		initializeSemaphore();
		this.fetchId = "#global#";
	}
	
	// Extension2 interface
	@Override
	public void registerExtensions(ExtensionManager mgr) {
		final int REQUIRED_REVISION = 100909;
		if (ExtUtil.REVISION < REQUIRED_REVISION) {
			Logger.warning(LOG_PREFIX +
					": ERROR!!! ExtUtil_"+REQUIRED_REVISION+" OR LATER IS REQUIRED.");
		} else {
			if (util == null) {
				util = new ExtUtil(this, LOG_PREFIX, PROP_DEBUG);
				util.loadFile(fetchResultsFileInfo, "loadJSON", FILE_CHARSET);
				util.execute("taker");
			}
			mgr.registerProcessor(this);
			mgr.registerRequestFilter(this);
		}
	}
	
	@Override
	public String getVersionString() {
		return VER_STRING;
	}
	
	// Processor Interface
	@Override
	public String[] getSupportedMethods() {
		return PROCESSOR_SUPPORTED_METHODS;
	}
	
	@Override
	public Pattern getSupportedURLAsPattern() {
		return PROCESSOR_SUPPORTED_PATTERN;
	}
	
	@Override
	public String getSupportedURLAsString() {
		return null;
	}
	
	@Override
	public Resource onRequest(HttpRequestHeader requestHeader, Socket browser)
			throws IOException {
		if (!Boolean.getBoolean("cacheThumbnail")) {
			util.info("must do 'cacheThumbnail=true' for using this extension.");
		} else {
			String method = requestHeader.getMethod();
			String fetch_uri = requestHeader.getURI();
			util.debug("%s %s", method, fetch_uri);
			Matcher m = PROCESSOR_SUPPORTED_PATTERN.matcher(fetch_uri);
			if (m.matches()) {
				String param = m.group(1);
				requestHeader.setParameter(FETCH_URI, fetch_uri);
				if (method.equals("GET")) {
					return procRequestGET(param, requestHeader);
				} else if (method.equals("POST")) {
					InputStream in = browser.getInputStream();
					byte[] buf = new byte[(int)requestHeader.getContentLength()];
					for (int readed = 0; readed < buf.length;)
						readed += in.read(buf, readed, buf.length - readed);
					String postBody = new String(buf);
					debug2("%s", postBody);
					return procRequestPOST(param, postBody);
				}
			}
		}
		return StringResource.getNotFound();
	}
	
	private StringResource procRequestGET(String param, HttpRequestHeader requestHeader) {
		StringResource result = resultNG;
		if (param.equals("all")) {
			// DO NOTHING
		} else if (param.startsWith("status=")) {
			result = method_status(param.substring(7));
		} else if (param.startsWith("rm=")) {
			result = method_remove(param.substring(3), requestHeader);
		} else if (param.startsWith("cancel=")) {
			result = method_cancel(param.substring(7));
		} else if (param.startsWith("mylist=")) {
			result = method_mylist(param.substring(7), requestHeader);
		} else if (param.equals("list")) {
			result = method_list();
		} else if (param.equals("json")) {
			result = method_json();
		} else if (param.equals("storage")) {
			result = method_storage_get();
		} else {
			result = method_fetch(param, requestHeader);
		}
		return result;
	}
	
	private StringResource procRequestPOST(String param, String postBody) {
		StringResource result = resultNG;
		if (param.equals("storage")) {
			result = method_storage_post(postBody);
		}
		return result;
	}
	
	// RequestFilter interface
	@Override
	public int onRequest(HttpRequestHeader requestHeader) throws IOException {
		String uri = requestHeader.getURI();
		boolean autoWatched = Boolean.getBoolean("movieFetchAutoWatched");
		Matcher m;
		if (uri.equals("http://ext.nicovideo.jp/thumb_watch")) {
			overwriteUserSession(requestHeader);
		} else if (autoWatched && (m = SM_FLV_PATTERN.matcher(uri)).find()) {
			markWatched(m.group(2));
		} else if (autoWatched && (m = DMC_CONTENT_URI_PATTERN.matcher(uri)).find()) {
			markWatched(m.group(2));
		} else if ((m = OTHER_SUBDOMAIN_PATTERN.matcher(uri)).matches()) {
			requestHeader.setURI("http://www" + m.group(1));
			requestHeader.setMessageHeader("Host", "www.nicovideo.jp");
		}
		return RequestFilter.OK;
	}
	
	private final Pattern OTHER_SUBDOMAIN_PATTERN = Pattern.compile(
			"http://(?!www\\.)[^/]+(\\.nicovideo\\.jp/cache/fetch\\?.*)");
	
	private void markWatched(String id) {
		String smid = Cache.id2Smid(id);
		if (smid != null && fetchResults.containsKey(smid)) {
			fetchResults.remove(smid);
			fetchResultsUpdated = true;
			util.debug("%s results auto removed.", smid);
		}
	}
	
	// TransferListener interface
	@Override
	public void onResponseHeader(HttpResponseHeader responseHeader) {
		// selectTransferTo(,,,false)を使っているのでここは呼ばれない
		// contentLengthはfetch()の中で設定
	}
	
	@Override
	public void onTransferBegin(OutputStream receiverOut) {
		transferedLength = 0L;
		if(contentLength > 0L) setStatusMessage("0%", true);
	}
	
	@Override
	public void onTransferring(byte[] buf, int length) {
		transferedLength += (long)length;
		if (contentLength > 0L) {
			int done = (int)(transferedLength * 100L / contentLength);
			setStatusMessage(done + "%", true);
		}
	}
	
	@Override
	public void onTransferEnd(boolean completed) {
		if (contentLength > 0L) {
			setStatusMessage(completed ? "done." : "FAILED!", true);
		}
		debugWithId("content=" + contentLength + ", transfered=" + transferedLength);
	}
	
	private String statusMessage = "not initialized.";
	private String getStatusMessage() {
		return statusMessage;
	}
	
	private void setStatusMessage(String status) {
		setStatusMessage(status, false);
	}
	private void setStatusMessage(String status,  boolean prefix) {
		statusMessage = prefix ? "fetching... ".concat(status) : status;
	}
	
	private void infoStatusMessage(String status) {
		if(status != null) setStatusMessage(status);
		util.info(fetchId + " " + statusMessage);
	}
	
	// ExtUtil callback
	public void onMinutes() {
		expireFetchRequests();
		onShutdown();
	}
	
	public void onShutdown() {
		if (fetchResultsUpdated) {
			fetchResultsUpdated = false;
			util.saveFile(fetchResultsFileInfo, createJSON(), true, FILE_CHARSET);
		}
		if (storageUpdated) {
			storageUpdated = false;
			util.saveFile(storageFileInfo, storageCache, true, FILE_CHARSET);
		}
	}
	
	private static Charset getCharset(String propertyName) {
		Charset charset = Charset.defaultCharset();
		String charsetName = System.getProperty(propertyName);
		if (charsetName != null && !charsetName.equals("")) {
			charset = Charset.forName(charsetName);
		}
		return charset;
	}
	
	private static int maxFetch = 0;
	private static int maxFetchLocked = 0;
	private static Semaphore sem = null;
	private static int fetchCount() { return maxFetch - sem.availablePermits(); }
	private static ScheduledExecutorService sched;
	private static final Random rand = new Random();
	private static final AtomicInteger accessRestriction = new AtomicInteger();
	private static final LinkedBlockingQueue<nlMovieFetcher> requestQueue =
			new LinkedBlockingQueue<nlMovieFetcher>();
	private static final Map<String, nlMovieFetcher> fetchRequests =
			Collections.synchronizedMap(new java.util.LinkedHashMap<String, nlMovieFetcher>());
	
	// ステータス文字列オブジェクト(主に比較が必要なものだけ)
	private static final String STATUS_FAILED    = "failed.";
	private static final String STATUS_RESERVED  = "reserved.";
	private static final String STATUS_CANCELED  = "canceled.";
	private static final String STATUS_NOT_FOUND = "not found.";
	private static final String STATUS_STREAMING = "streaming.";
	private static final String STATUS_NO_DISK   = "no disk free space.";
	
	private int maxRetry = ExtUtil.getInteger("movieFetchRetry", 5, 0, 9);
	private String status = STATUS_FAILED;
	
	// コンストラクタ(各タスク用)
	private nlMovieFetcher(String id, HttpRequestHeader requestHeader) {
		this.fetchId = id;
		this.fetchRequestHeader = requestHeader;
		setStatusMessage("wait", true);
	}
	
	// fetch処理は別スレッドで実行
	@Override
	public void run() {
		if (isHearbeatThread()) {
			runHeartbeat();
			return;
		}
		if (fetchRequestHeader.getMessageHeader("Cookie") != null) {
			fetchRequestHeader.setMessageHeader("Cookie", fetchRequestHeader.getMessageHeader("Cookie").replace("watch_html5=1", "watch_html5=0"));
		}
		started = true;
		debugWithId("task started.");
		if(updatelow()) return;
		
		String smid = id2smid(fetchId, null);
		util.debug("id2smid(fetchId, null): %s %s", fetchId, smid);
		for (int retry = 0; retry <= maxRetry; retry++) {
			if (!acquireSemaphore()) {
				setStatusWithoutRetry(STATUS_CANCELED); break; // FAILURE
			}
			captureUserSession(fetchRequestHeader.getMessageHeader("Cookie"),
					System.getProperty("movieFetchCaptureUserID", "").trim());
			overwriteUserSession(fetchRequestHeader);
			try {
				if (!avoidMultipleAccess()) {
					setStatusWithoutRetry(STATUS_CANCELED); break; // FAILURE
				}
				setStatusMessage("watch page, wait", true);
				URLResource r = getWatchPageResource();
				if(r == null) break; // FAILURE
				HttpResponseHeader header = r.getResponseHeader(null, null);
				byte[] body = r.getResponseBody();
				if (header != null && body != null) {
					String content = new String(body, ExtUtil.getCharset(header));
					if (header.getStatusCode() == 200) {
						String authflag = header.getMessageHeader("x-niconico-authflag");
						if (authflag == null || authflag.equals("0")) {
							debugWithId("unauthorized, trying to login...");
							if (login(fetchRequestHeader)) {
								continue; // ログイン成功ならもう一度
							} else {
								setStatusWithoutRetry("unauthorized."); break;
							}
						}
						if (smid.matches("\\d+")) {
							// コミュニティの場合はgetthumbinfoで取得出来ないので
							// 改めてwatchページから取得する
							smid = id2smid(fetchId, content);
						}
						setStatusMessage("content, wait", true);
						procWatch(header, smid, content);
						if (status == STATUS_RESERVED) {
							break;
						}
						if (Cache.getPreferredCachedVideo(smid) != null) {
							break; // SUCCEEDED
						}
						else if (Boolean.getBoolean("checkRealCache")) {
							// dareka/processor/impl/CacheManager.javaのid2File_get()を
							// 見る限りでは、checkRealCacheは正しく扱われない模様
							// [dmc] 無意味な処理っぽいけど手動でキャッシュフォルダに
							//   動画を追加を追加したせいで処理に失敗したような
							//   特殊な状況では有益かも
							Cache.init();
							if (Cache.getPreferredCachedVideo(smid) != null) {
								break; // SUCCEEDED
							}
						}
					} else {
						procError(header.getStatusCode(), content);
					}
				}
			} catch (Exception e) {
				Logger.error(e);
			} finally {
				sem.release();
			}
			if (retry < maxRetry) {
				if (!waitForRetry(smid, retry)) {
					setStatus(STATUS_CANCELED); break;
				}
			}
		}
		if (status == STATUS_RESERVED) {
			return; // 予約の場合はもう一度実行するのでここで終了
		}
		if (Cache.getPreferredCachedVideo(smid) != null) {
			infoStatusMessage("done.");
		}
		else {
			updateFetchResult(smid, fetchId + " " + status);
			infoStatusMessage(status);
		}
		completedTime = System.currentTimeMillis();
		debugWithId("task completed.");
	}
	
	private URLResource getWatchPageResource() throws IOException {
		String watch_url = "http://www.nicovideo.jp/watch/".concat(fetchId);
		URLResource r = new URLResource(watch_url);
		r.setFollowRedirects(false); // リダイレクトを自前で処理する
		HttpResponseHeader responseHeader = r.getResponseHeader(
				null, ExtUtil.setURL(fetchRequestHeader, watch_url, false));
		if (responseHeader != null && ( (responseHeader.getStatusCode() / 100) ) == 3) { /* 121015 FIX */
			String location = responseHeader.getMessageHeader("Location");
			if (location != null) {
				if (location.contains("/ppv_video/")) {
					setStatusWithoutRetry("ppv_video.");
					return null; // FAILURE
				}
				if (location.startsWith("/")) {
					watch_url = "http://www.nicovideo.jp".concat(location);
				} else {
					watch_url = location;
				}
				r = new URLResource(watch_url);
				responseHeader = r.getResponseHeader(
						null, ExtUtil.setURL(fetchRequestHeader, watch_url, false));
			} else {
				setStatus("redirect error.");
				return null; // FAILURE
			}
		}
		return r; // レスポンスヘッダ受信済
	}
	
	private void procWatch(
			HttpResponseHeader responseHeader, String smid, String content) {
		accessRestriction.set(0); // 連続アクセス規制解除
		if (!setNicohistoryCookie(responseHeader)) {
			setStatusWithoutRetry("cookie error.");
			return;
		}
		// 先にnl本体にタイトルを認識させておく(公式動画では必須)
		String title = recognizeTitle(smid, content);
		
		WatchVars wv = getWatchVars(smid, content);
		if (wv.isPremium()) {
			premiumWait = Integer.getInteger("movieFetchPremiumWait", 0);
			unlockPremiumSemaphore();
		}
		JsonObject json = wv.getJsonObject();
		String dmcInfoString = json != null ? json.getString("flashvars", "dmcInfo") : null;
		String flvInfo = null;
		
		Boolean reencoded = true;
		if (Boolean.getBoolean("movieFetchCheckReEncoding")) {
			reencoded = ReEncodingInfo.get(smid);
			if (dmcInfoString != null && reencoded == null) {
				// detect reencoding if dmc exists
				flvInfo = getFlvInfo(content, json);
				if (flvInfo == null) {
					setStatus(STATUS_FAILED);
					return;
				}
				reencoded = procDetectReEncoding(smid, flvInfo);
				if (reencoded == null) {
					// statusはセットされている
					return;
				}
			}
		}
		
		if (dmcInfoString != null && reencoded != null && reencoded) {
			// dmc
			try {
				dmcInfoString = URLDecoder.decode(dmcInfoString, "UTF-8");
			} catch (UnsupportedEncodingException e) {
				Logger.error(e);
				setStatusWithoutRetry(STATUS_FAILED);
				return;
			}
			JsonObject dmcInfo = Json.parseObject(dmcInfoString);
			procDmc(smid, dmcInfo, title);
		} else {
			// smile
			if (flvInfo == null) {
				flvInfo = getFlvInfo(content, json);
				if (flvInfo == null) {
					setStatus(STATUS_FAILED);
					return;
				}
			}
			procSmile(smid, flvInfo, title);
		}
	}
	
	private Boolean procDetectReEncoding(String smid, String flvInfo) {
		String movie_url = getMovieUrlFromFlvInfo(flvInfo);
		if (movie_url == null) {
			setStatusWithoutRetry(STATUS_FAILED);
			return null;
		} else if (movie_url.startsWith("rtmp")) {
			setStatusWithoutRetry(STATUS_STREAMING);
			return null;
		} else if (!movie_url.startsWith("http://")) {
			// deleted or unknown scheme
			setStatusWithoutRetry(movie_url);
			return null;
		}
		VideoDescriptor video = determineVideoDescriptorFromSmileUrl(smid, movie_url);
		if (video == null) {
			setStatusWithoutRetry(STATUS_FAILED);
			return null;
		}
		if (Boolean.getBoolean("movieFetchNoEconomy") && video.isLow()) {
			reserveFetchRequest(this);
			setStatus(STATUS_RESERVED);
			return null;
		}
		if (!Cache.MP4.equals(video.getPostfix())) {
			return false;
		}
		byte[] videoHeader = fetchVideoHeader(movie_url, fetchRequestHeader);
		if (videoHeader == null) {
			setStatusWithoutRetry(STATUS_FAILED);
			return null;
		}
		Boolean result = ReEncodingInfo.check(video.getId(), ByteBuffer.wrap(videoHeader));
		if (result == null) {
			setStatusWithoutRetry(STATUS_FAILED);
			return null;
		}
		return result;
	}
	
	private void procSmile(String smid, String flvInfo, String title) {
		String movie_url = getMovieUrlFromFlvInfo(flvInfo);
		if (movie_url == null) {
			setStatusWithoutRetry(STATUS_FAILED);
			return;
		} else if (movie_url.startsWith("rtmp")) {
			setStatusWithoutRetry(STATUS_STREAMING);
			return;
		} else if (!movie_url.startsWith("http://")) {
			// deleted or unknown scheme
			setStatusWithoutRetry(movie_url);
			return;
		}
		VideoDescriptor video = determineVideoDescriptorFromSmileUrl(smid, movie_url);
		if (video == null) {
			setStatusWithoutRetry(STATUS_FAILED);
			return;
		}
		if (Boolean.getBoolean("movieFetchNoEconomy") && video.isLow()) {
			reserveFetchRequest(this);
			setStatus(STATUS_RESERVED);
			return;
		}
		String result = fetch(movie_url, fetchRequestHeader);
		if (result.startsWith("http://")) {
			updateFetchResult(smid, title); // とりあえず結果を保存
			waitForComplete(video);
			setStatus(result);
		} else if (result != STATUS_NOT_FOUND) {
			setStatusWithoutRetry(result);
		} else {
			setStatus(STATUS_NOT_FOUND);
		}
	}
	
	private void procDmc(String smid, JsonObject dmcInfo, String title) {
		dmcRequestHeader = cloneHttpRequestHeader(fetchRequestHeader);
		if (dmcRequestHeader == null) {
			setStatusWithoutRetry(STATUS_FAILED);
			return;
		}
		dmcRequestHeader.removeMessageHeader("Cookie");
		dmcRequestHeader.setMessageHeader("Origin", "http://www.nicovideo.jp");
		dmcApiRequestHeader = cloneHttpRequestHeader(dmcRequestHeader);
		if (dmcApiRequestHeader == null) {
			setStatusWithoutRetry(STATUS_FAILED);
			return;
		}
		dmcApiRequestHeader.setMessageHeader("accept", "application/json");
		
		dmcApiUrl = dmcInfo.getString("session_api", "api_urls", 0);
		if (dmcApiUrl == null) {
			setStatusWithoutRetry(STATUS_FAILED);
			return;
		}
		
		dmcApiData = createDmcSession(dmcInfo, dmcApiRequestHeader);
		if (dmcApiData == null) {
			setStatusWithoutRetry(STATUS_FAILED);
			return;
		}
		VideoDescriptor video = determineVideoDescriptorFromDmcSession(smid, dmcApiData);
		if (video == null) {
			setStatusWithoutRetry(STATUS_FAILED);
			// セッションを削除してないけど良いよね……
			return;
		}
		if (Boolean.getBoolean("movieFetchNoEconomy") && video.isLow()) {
			reserveFetchRequest(this);
			setStatus(STATUS_RESERVED);
			return;
		}
		
		Thread heartbeat = createDmcHeartbeatThread();
		heartbeat.start();
		try {
			String movieUrl = getMovieUrlFromDmcApiData(dmcApiData);
			if (movieUrl == null) {
				setStatusWithoutRetry(STATUS_FAILED);
				return;
			}
			String result = fetch(movieUrl, dmcRequestHeader);
			if (result.startsWith("http://")) {
				updateFetchResult(smid, title); // とりあえず結果を保存
				waitForComplete(video);
				setStatus(result);
			} else if (result != STATUS_NOT_FOUND) {
				setStatusWithoutRetry(result);
			} else {
				setStatus(STATUS_NOT_FOUND);
			}
		} finally {
			shutdownDmcHeartbeatThread(heartbeat);
		}
	}
	
	private void procError(int statusCode, String content) {
		if (statusCode == 403) {
			if (content.contains("連続アクセス")) {
				accessRestriction.incrementAndGet();
			} else if (content.contains("非表示")) {
				setStatusWithoutRetry("currently closed.");
			} else if (content.contains("視聴権限")) {
				setStatusWithoutRetry("member only.");
			} else if (content.contains("削除")) {
				setStatusWithoutRetry("deleted.");
			} else {
				setStatus("forbidden.");
			}
		} else if (statusCode == 404) {
			// 404の場合はリトライしても無駄っぽいので終了
			if (content.contains("<span id=\"deleted_message_")) {
				setStatusWithoutRetry("deleted.");
			} else {
				setStatusWithoutRetry("not found.");
			}
		}
	}
	
	private void setStatusWithoutRetry(String status) {
		this.status = status;
		this.maxRetry = 0;
	}
	
	private void setStatus(String status) {
		this.status = status;
	}
	
	private static final Pattern NICOHISTORY_PATTERN = Pattern.compile(
			"Set-Cookie\\s*?:\\s*?(nicohistory=[^;]+?);"); /* 121015 FIX */
	private boolean setNicohistoryCookie(HttpResponseHeader responseHeader) {
		// String set_cookie = responseHeader.getMessageHeader("Set-Cookie"); /* 121015 FIX */
		// if (set_cookie != null) {
			Matcher m = NICOHISTORY_PATTERN.matcher(responseHeader.toString());
			if (m.find()) {
				String cookie = fetchRequestHeader.getMessageHeader("Cookie");
				if (cookie == null) {
					cookie = m.group(1);
				} else {
					if (cookie.indexOf("nicohistory=") < 0) {
						cookie += "; " + m.group(1);
					} else {
						cookie = cookie.replaceFirst("nicohistory=[^;]+", m.group(1));
					}
				}
				fetchRequestHeader.setMessageHeader("Cookie", cookie);
				return true;
			}
		//} /* 121015 FIX */
		util.info("cannot get nicohistory cookie.");
		debug2("%s", responseHeader);
		return false;
	}
	
	private static final Pattern WATCH_URL_V_PATTERN = Pattern.compile(
			"^http://www\\.nicovideo\\.jp/watch/(\\w{2}\\d+)");
	private static final Pattern WATCHPAGE_V_PATTERN = Pattern.compile(
			"((?<=\\.addVariable\\()\"|&quot;)v\\1\\s*?[,:]\\s*?\\1(\\w{2}\\d+)\\1"); /* 121016 FIX */
	private static final Pattern WATCHPAGE_FLVPLAYER_PATTERN = Pattern.compile(
			"new\\s+SWFObject\\(\"(http://[^\"]+)\",\\s*\"flvplayer\",");
	private static final Pattern WATCHPAGE_MOVIETYPE_PATTERN = Pattern.compile(
			"((?<=\\.addVariable\\()\"|&quot;)movie_type\\1\\s*?[,:]\\s*?\\1([^\"&]+?)\\1"); /* 121016 FIX */
	
	private String getflv(String content) {
		String getflv_url = "http://flapi.nicovideo.jp/api/getflv";
		String v = fetchId, post_string = null;
		long ts = System.currentTimeMillis();
		if (content == null) {
			Matcher m = WATCH_URL_V_PATTERN.matcher(fetchRequestHeader.getURI());
			if (m.find()) {
				v = m.group(1);
			}
			getflv_url += "/" + v + "?as3=1&ts=" + ts;
		} else {
			Matcher m = WATCHPAGE_V_PATTERN.matcher(content);
			if (m.find()) {
				v = m.group(2);
			}
			post_string = "ts=" + ts + "&v=" + v;
			m = WATCHPAGE_FLVPLAYER_PATTERN.matcher(content);
			if (m.find()) {
				fetchRequestHeader.setURI(m.group(1)); // Referer
			}
			m = WATCHPAGE_MOVIETYPE_PATTERN.matcher(content);
			if (m.find() && m.group(2).equals("swf")) {
				post_string += "&as3=1";
			}
		}
		String getflvContent = ExtUtil.http_get(
				ExtUtil.setURL(fetchRequestHeader, getflv_url, true), post_string);
		debug2("requestHeader:\n%spostString: %s\n\ngetflvContent: %s\n",
				fetchRequestHeader, post_string, getflvContent);
		return getflvContent;
	}
	
	private static synchronized void initializeSemaphore() {
		if (sem != null) {
			return;
		}
		int trueMaxFetch =  ExtUtil.getInteger("movieFetchMax", 5, 1, 5);
		sched = Executors.newScheduledThreadPool(trueMaxFetch * 2);
		if (trueMaxFetch > 2) {
			maxFetch = 2;
			maxFetchLocked = trueMaxFetch - maxFetch;
		} else {
			maxFetch = trueMaxFetch;
			maxFetchLocked = 0;
		}
		sem = new Semaphore(maxFetch, true);
	}
	
	private static synchronized void unlockPremiumSemaphore() {
		sem.release(maxFetchLocked);
		maxFetchLocked = 0;
	}
	
	private boolean acquireSemaphore() {
		try {
			if (this.canceled) {
				throw new InterruptedException();
			}
			setStatusMessage("thread stopped. wait");
			sem.acquire();
			setStatusMessage("thread started. wait");
		} catch(InterruptedException e) {
			debugWithId("interrupted.");
			return false;
		}
		return true;
	}
	
	private boolean waitForRetry(String smid, int retryCount) {
		String reason = "failed";
		int wait_count = 10 * (retryCount + 1);
		int n = accessRestriction.get();
		if (n > 0) {
			reason = "forbidden";
			wait_count = wait_count * 30 + rand.nextInt(n) * n + n;
		}
		util.info(smid+" "+reason+", waiting "+wait_count+" seconds for next retry...");
		setStatusMessage("retrying(" + retryCount + ")... wait");
		return secWait(wait_count);
	}
	
	private void waitForComplete(VideoDescriptor video) {
		// 割り込みをクリアしておく
		Thread.interrupted();
		
		// 少し待たないとキャッシュファイルが出来ていない事があるので…
		do { secWait(2); } while (Cache.getDLFlag(video));
		
		if (premiumWait > 0) {
			int wait = rand.nextInt(premiumWait) + 1;
			debugWithId("premium waiting " + wait + " sec.");
			secWait(wait);
		}
	}
	
	private boolean secWait(int sec) {
		if (sec > 0) {
			try {
				Thread.sleep(sec * 1000);
			} catch(InterruptedException e) {
				debugWithId("interrupted.");
				return false;
			}
		}
		return true;
	}
	
	private void updateFetchResult(String smid, String title) {
		if (!smid.matches("[a-z]{2}\\d+"))
			return;
		if (fetchRequestHeader != null &&
				fetchRequestHeader.getParameter(FETCH_URI).contains("&result=0"))
			return;
		fetchResults.remove(smid); // 一度削除してリストの最後にする
		fetchResults.put(smid, title);
		fetchResultsUpdated = true;
	}
	
	// キューからfetcherを取り出して実行
	private static int continuousRequestCount = 0;
	public void taker() {
		try {
			for (;;) {
				nlMovieFetcher fetcher = requestQueue.take(); // BLOCK
				long delay = 0;
				if (!Boolean.getBoolean("movieFetchNoDelay")) {
					int wait_count = fetchCount() + 1;
					// 連投を確認するためにしばらく待つ
					util.debug("%s initial waiting %d sec.",
							fetcher.fetchId, wait_count);
					Thread.sleep(wait_count * 1000);
					delay = continuousRequestCount++ * wait_count * 1500;
					if (requestQueue.size() == 0) {
						continuousRequestCount = 0;
					}
				}
				scheduleFetchRequest(fetcher, delay);
			}
		} catch (InterruptedException e) {
			Logger.error(e);
		}
	}
	
	// updatelow.rb相当の機能
	private static final Pattern TEMPLIST_JSON_PATTERN = Pattern.compile(
			"^\"([a-z]{2}\\d+)(?:low)?\":\\[\"([^\"]*)\",");
	private static final int MIN_INTERVAL = 10000;
	private boolean updatelow() {
		if (!canceled && fetchId.equals("updatelow")) {
			infoStatusMessage("started.");
			long delay = 0;
			for (VideoDescriptor video : Cache.getVideo2File().keySet()) {
				if (video.isLow()) {
					String title = new Cache(video).getTitle();
					if (title.matches("削除\\]$")) continue;
					scheduleFetchRequest(video.getId(), delay);
					delay += MIN_INTERVAL + rand.nextInt(MIN_INTERVAL * 2);
				}
			}
			// 一時ファイルは直接取れないのでJSONで取得
			for (String line : Cache.getTempListAsJson(false).split("\n")) {
				Matcher m = TEMPLIST_JSON_PATTERN.matcher(line);
				if (m.lookingAt()) {
					scheduleFetchRequest(m.group(1), delay);
					delay += MIN_INTERVAL + rand.nextInt(MIN_INTERVAL);
				}
			}
			// 削除以外でfetchに失敗したものも再取得
			synchronized (fetchResults) {
				for (Map.Entry<String,String> e : fetchResults.entrySet()) {
					String smid = e.getKey(), reason = e.getValue();
					if (Cache.getPreferredCachedVideo(smid) == null &&
							reason.indexOf("deleted") < 0) {
						scheduleFetchRequest(smid, delay);
						delay += MIN_INTERVAL + rand.nextInt(MIN_INTERVAL * 3);
					}
				}
			}
			completedTime = System.currentTimeMillis();
			infoStatusMessage("done.");
			debugWithId("task completed.");
			return true; // "updatelow"を処理した
		}
		if (Boolean.getBoolean("movieFetchUpdateCache")) {
			if (fetchRequests.containsKey("updatelow")) {
				// updatelowが予約中/実行中である
				if (Boolean.getBoolean("movieFetchNoDelay")) {
					// updatelowが予約の場合のみ、予約を取り消して即座に実行する
					nlMovieFetcher updatelow = fetchRequests.get("updatelow");
					if (updatelow != null && updatelow.future != null) {
						if (updatelow.future.getDelay(TimeUnit.MILLISECONDS) > 0) {
							updatelow.future.cancel(false);
							fetchRequests.remove("updatelow");
							scheduleFetchRequest("updatelow", 0);
						}
					}
				}
			}
			else {
				// updatelowが実行中でも、予約中でも無い
				if (Boolean.getBoolean("movieFetchNoDelay")) {
					scheduleFetchRequest("updatelow", 0);
				}
				else {
					reserveFetchRequest("updatelow");
				}
			}
		}
		return false;
	}
	
	private StringResource method_remove(String id, HttpRequestHeader requestHeader) {
		boolean result = true;
		if (id.equals("all")) {
			synchronized (fetchResultsFileInfo) {
				fetchResults.clear();
				ExtUtil.renameToBak(fetchResultsFileInfo.first);
			}
		} else {
			result = fetchResults.remove(id2smid(id, null)) != null;
		}
		if (result) {
			fetchResultsUpdated = true;
			util.debug("%s results removed.", id);
			return resultOK;
		}
		return resultNG;
	}
	
	private StringResource method_cancel(String id) {
		if (id.equals("all")) {
			synchronized (fetchRequests) {
				for (nlMovieFetcher fetcher : fetchRequests.values()) {
					cancelFetchRequest(fetcher);
				}
			}
			return resultOK;
		} else {
			nlMovieFetcher fetcher = fetchRequests.get(id);
			if (fetcher != null) {
				cancelFetchRequest(fetcher);
				return resultOK;
			}
		}
		return resultNG;
	}
	
	private StringResource method_mylist(String id, HttpRequestHeader requestHeader) {
		String content = null;
		
		try {
			if (id.equals("all")) {
				content = getPageContent(URL_MYLISTGROUP_API, requestHeader);
				if (content != null) {
					Matcher m = JSON_MYLIST_ID_PATTERN.matcher(content);
					long wait = 0;
					while (m.find()) {
						if (method_mylist(m.group(1), requestHeader).equals(resultNG)) {
							return resultNG;
						}
						Thread.sleep(wait++ * 1000); // "短時間での連続アクセスはご遠慮ください"
					}
				}
				util.info("deflist");
				content = getPageContent(URL_DEFLIST_API, requestHeader);
			}
			else {
				util.info("mylist %s", id);
				content = getPageContent(URL_MYLIST + id, requestHeader, true);
			}
		}
		catch (Exception e) {
			Logger.error(e);
		}
		if (content == null) {
			util.info("method_mylist(): cannot get content from %s", id);
			return resultNG;
		}
		else {
			Matcher m = JSON_VIDEO_ID_PATTERN.matcher(content);
			String video_id = null;
			while (m.find()) {
				video_id = m.group(1);
				util.debug("video_id %s", video_id);
				if (method_fetch(video_id, requestHeader).equals(resultNG)) {
					return resultNG;
				}
			}
		}
		
		return ExtUtil.responseText("OK fetch done.");
	}
	
	private String getPageContent(String url) throws IOException {
		return getPageContent(url, fetchRequestHeader, false);
	}
	
	private String getPageContent(String url, HttpRequestHeader requestHeader) throws IOException {
		return getPageContent(url, requestHeader, false);
	}
	
	private String getPageContent(String url, HttpRequestHeader requestHeader,
		boolean checkbody) throws IOException {
		String body = null;
		
		URLResource r = new URLResource(url);
		r.setFollowRedirects(true); // リダイレクトを自動で追跡する
		HttpResponseHeader responseHeader = r.getResponseHeader(
			null, ExtUtil.setURL(requestHeader, url, false));
		if (responseHeader != null && responseHeader.getStatusCode() == 200) {
			body = new String(r.getResponseBody(), "UTF-8");
			if (checkbody && Pattern.compile("短時間での連続アクセスはご遠慮ください").matcher(body).find()) {
				return null;
			}
		}
		return body;
	}
	
	private StringResource method_list() {
		return ExtUtil.responseText(
				"var fetchedList = "+createJSON()+";", "text/javascript");
	}
	
	private StringResource method_json() {
		return ExtUtil.responseText(createJSON(), "application/json");
	}
	
	private StringResource method_storage_get() {
		if (storageCache == null || !storageUpdated) {
			util.loadFile(storageFileInfo, "loadStorage", FILE_CHARSET);
		}
		return ExtUtil.responseText(storageCache, "application/json");
	}
	
	public void loadStorage(BufferedReader br) throws IOException {
		if (br != null) {
			long len = storageFileInfo.first.length();
			if (len > Integer.MAX_VALUE) {
				util.warn("'%s' is too large (%,d MB), skip.",
						storageFileInfo.first, len / (1024 * 1024));
				return;
			}
			StringBuilder sb = new StringBuilder((int)len);
			String line;
			while ((line = br.readLine()) != null) {
				sb.append(line);
			}
			storageCache = sb.toString();
		} else {
			storageCache = "{}";
		}
	}
	
	private StringResource method_storage_post(String storageNew) {
		String storageOld = storageCache;
		storageCache = storageNew;
		if (storageOld == null || storageOld.length() != storageNew.length() ||
				!storageOld.equals(storageNew)) {
			storageUpdated = true;
		}
		return resultOK;
	}
	
	private StringResource method_status(String id) {
		if (id.equals("all")) {
			StringBuilder sb = new StringBuilder(fetchRequests.size() * 40);
			DateFormat df = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
			long delay;
			nlMovieFetcher fetcher;
			synchronized (fetchRequests) {
				for (Map.Entry<String, nlMovieFetcher> e : fetchRequests.entrySet()) {
					// <id>: <statusMessage>CRLF
					fetcher = e.getValue();
					sb.append(e.getKey());
					sb.append(": ");
					sb.append(fetcher.getStatusMessage());
					if (fetcher.future != null) {
						delay = fetcher.future.getDelay(TimeUnit.MILLISECONDS);
						if (delay > 0 && ! fetcher.canceled) {
							Calendar c = Calendar.getInstance();
							c.setTimeInMillis( c.getTimeInMillis() + delay );
							sb.append(" ("+ df.format(c.getTime()) +")");
						}
					}
					sb.append("\r\n");
				}
			}
			return ExtUtil.responseText(sb.toString());
		} else {
			nlMovieFetcher fetcher = fetchRequests.get(id);
			if (fetcher != null) {
				String message = fetcher.getStatusMessage();
				if (message != null) {
					return ExtUtil.responseText(message);
				}
			}
		}
		return StringResource.getNotFound();
	}
	
	private StringResource method_fetch(String id, HttpRequestHeader requestHeader) {
		if (id.equals("updatelow")) {
			nlMovieFetcher fetcher = fetchRequests.remove(id);
			if (fetcher != null && fetcher.future != null)
				fetcher.future.cancel(false);
		}
		else {
			String smid = id2smid(id, null);
			if (ExtUtil.hasRemoved(smid)) {
				return infoStringResourceOK(smid, "hasRemoved!");
			}
			String thumbinfo = null;
			boolean ng_title = Boolean.getBoolean("movieFetchUseNGtitle");
			boolean ng_user  = Boolean.getBoolean("movieFetchUseNGuser");
			if (ng_title || ng_user) {
				try {
					thumbinfo = GetThumbInfoUtil.get(smid);
				} catch (IOException e) {}
			}
			if (ng_title && thumbinfo != null) {
				Matcher m = THUMBINFO_TITLE_PATTERN.matcher(thumbinfo);
				if (m.find() && ExtUtil.matchNGtitle(m.group(1)) != null)
					return infoStringResourceOK(smid, "NGtitle!!");
			}
			if (ng_user && thumbinfo != null) {
				Matcher m = THUMBINFO_USERID_PATTERN.matcher(thumbinfo);
				if (m.find() && ExtUtil.isNGuserId(m.group(1)))
					return infoStringResourceOK(smid, "NGuser!!");
			}
			if (Cache.getPreferredCachedVideo(smid) != null) {
				return infoStringResourceOK(smid, "already cached.");
			} else if (Cache.getDLFlag(VideoDescriptor.newClassic(smid, Cache.FLV, false))) {
				return infoStringResourceOK(smid, "now caching.");
			}
		}
		nlMovieFetcher fetcher = addFetchRequest(id, requestHeader);
		if (fetcher == null) {
			return ExtUtil.responseText("OK already requested.");
		} else if (requestQueue.offer(fetcher)) {
			return ExtUtil.responseText("OK fetch accepted.");
		}
		return resultNG;
	}
	
	private StringResource infoStringResourceOK(String smid, String message) {
		util.info(smid + " " + message);
		return ExtUtil.responseText("OK ".concat(message));
	}
	
	private boolean checkDiskFreeSpace() {
		long freeSpace = DiskFreeSpace.get(Cache.getCacheDir());
		long needed = Long.getLong("needFreeSpace", 100) * 1024 * 1024;
		if (needed < 0 || freeSpace < needed)
			return false;
		return true;
	}
	
	private nlMovieFetcher addFetchRequest(String id, HttpRequestHeader requestHeader) {
		nlMovieFetcher fetcher = fetchRequests.get(id);
/*		if (fetcher != null && fetcher.completedTime > 0) {
			// 完了リクエストなら直ぐに削除して再度リクエストを受け付ける
			// ここに来るのはキャッシュが無い場合なので再度実行して問題無いはず
			fetchRequests.remove(id);
			fetcher = null;
		}
		if (fetcher == null) {
			fetcher = new nlMovieFetcher(id, requestHeader);
			fetchRequests.put(id, fetcher);
		} else {
			util.debug("%s request found.", id);
		}*/
		if (fetcher != null) {
			util.debug("%s request found.", id);
			if (fetcher.completedTime == 0L) {
				return null; // 完了していなければ追加しない
			} else {
				// 完了リクエストなら直ぐに削除して再度リクエストを受け付ける
				// ここに来るのはキャッシュが無い場合なので再度実行して問題無いはず
				fetchRequests.remove(id);
				util.debug("%s request removed.", id);
			}
		}
		fetcher = new nlMovieFetcher(id, requestHeader);
		fetchRequests.put(id, fetcher);
		return fetcher;
	}
	
	private static final long REQUEST_EXPIRED = 60 * 1000; // 60秒
	private void expireFetchRequests() {
		long current = System.currentTimeMillis();
		Set<Map.Entry<String,nlMovieFetcher>> s = fetchRequests.entrySet();
		synchronized (fetchRequests) {
			for (Iterator<Map.Entry<String,nlMovieFetcher>> it = s.iterator();
				it.hasNext();
			) {
				Map.Entry<String,nlMovieFetcher> e =
					(Map.Entry<String,nlMovieFetcher>) it.next();
				nlMovieFetcher fetcher = e.getValue();
				long completed = fetcher.completedTime;
				if (completed == 0 &&
					fetcher.future != null &&
					fetcher.future.isCancelled() &&
					! fetcher.started
				) {
					fetcher.infoStatusMessage(STATUS_CANCELED);
					fetcher.completedTime = current;
				}
				if (completed != 0 && current - completed > REQUEST_EXPIRED) {
					it.remove();
					util.debug("%s request expired.", e.getKey());
				}
			}
		}
	}
	
	private void reserveFetchRequest(String id) {
		reserveFetchRequest(addFetchRequest(id, fetchRequestHeader));
	}
	private void reserveFetchRequest(nlMovieFetcher fetcher) {
		Calendar c = Calendar.getInstance();
		int hour = c.get(Calendar.HOUR_OF_DAY);
		if (hour >= 2) c.add(Calendar.DAY_OF_WEEK, 1); // 翌日
		c.set(Calendar.HOUR_OF_DAY, 2); // 午前2時(エコノミー時間帯明け)
		c.set(Calendar.MINUTE, 0);
		if (!Boolean.getBoolean("movieFetchNoDelay")) {
			// 時間を2時〜5時の間に進める
			c.set(Calendar.SECOND, 0);
			c.add(Calendar.SECOND, rand.nextInt(3 * 3600));
		}
		fetcher.setStatusMessage(STATUS_RESERVED);
		long delay = c.getTimeInMillis() - System.currentTimeMillis();
		scheduleFetchRequest(fetcher, delay);
		DateFormat df = DateFormat.getDateTimeInstance(DateFormat.LONG, DateFormat.LONG);
		util.info(fetcher.fetchId+" reserved at "+df.format(c.getTime())+".");
	}
	
	private void scheduleFetchRequest(String id, long delay) {
		scheduleFetchRequest(addFetchRequest(id, fetchRequestHeader), delay);
	}
	private void scheduleFetchRequest(nlMovieFetcher fetcher, long delay) {
		if (fetcher == null) {
			util.warn("%s cannot schedule null fetcher!", this.fetchId);
			return;
		}
		if(fetcher.canceled || delay < 0) delay = 0;
		util.debug("%s scheduled (%,dms delay).", fetcher.fetchId, delay);
		fetcher.future = sched.schedule(fetcher, delay, TimeUnit.MILLISECONDS);
	}
	
	private void cancelFetchRequest(nlMovieFetcher fetcher) {
		util.debug("%s canceling", fetcher.fetchId);
		fetcher.canceled = true;
		if (fetcher.future != null) {
			if (fetcher.future.getDelay(TimeUnit.MILLISECONDS) > 0) {
				fetcher.setStatusMessage(STATUS_CANCELED);
				fetcher.completedTime = System.currentTimeMillis();
			}
			fetcher.future.cancel(true);
		}
	}
	
	private static final Pattern JSON_LINE_PATTERN = Pattern.compile(
			"^\"([a-z]{2}\\d+)\":\\s*\"(.*)\",?$");
	public void loadJSON(BufferedReader br) throws IOException {
		fetchResults.clear();
		if (br != null) {
			String line;
			while ((line = br.readLine()) != null) {
				Matcher m = JSON_LINE_PATTERN.matcher(line);
				if (m.matches()) {
					fetchResults.put(m.group(1), m.group(2).replace("\\", ""));
				}
			}
		}
	}
	
	private String createJSON(boolean stripCRLF) {
		// タイトルの最大値が99バイトらしいので1行平均60文字程度で初期化
		StringBuilder sb = new StringBuilder(fetchResults.size() * 60);
		sb.append(stripCRLF ? "{" : "{\r\n");
		if (!fetchResults.isEmpty()) {
			synchronized (fetchResults) {
				for (Map.Entry<String,String> e : fetchResults.entrySet()) {
					// "<smid>": "<title>",CRLF
					sb.append("\"");
					sb.append(e.getKey());
					sb.append("\": \"");
					sb.append(e.getValue().replace("\"", "\\\""));
					sb.append(stripCRLF ? "\"," : "\",\r\n");
				}
			}
			sb.deleteCharAt(sb.lastIndexOf(","));
		}
		sb.append("}");
		return sb.toString();
	}
	
	private String createJSON() {
		return createJSON(false);
	}
	
	private static final Pattern WATCHPAGE_VIDEOID_PATTERN = Pattern.compile( /* 20130114 FIX */
			"(?s)(?:var Video = \\{|<div\\s+id=\"watchAPIDataContainer\").*?(&quot;|\"|'|)v\\1\\s*:\\s*(&quot;|\"|'|)(\\d+)\\2.+?(&quot;|\"|'|)id\\4\\s*:\\s*(&quot;|\"|')([a-z]{2}\\d+)\\5");
	private static final Pattern THUMBINFO_TITLE_PATTERN = Pattern.compile(
			"<title>([^<]+)</title>");
	private static final Pattern THUMBINFO_VIDEOID_PATTERN = Pattern.compile(
			"<video_id>([a-z]{2}\\d+)</video_id>");
	private static final Pattern THUMBINFO_USERID_PATTERN = Pattern.compile(
			"<user_id>(\\d+)</user_id>");
	
	private String id2smid(String id, String content) {
		String smid = id;
		// smidがthread_idの場合はvideo_idを取得
		if (id.matches("\\d+")) {
			if (content != null) { // watchページ
				Matcher m = WATCHPAGE_VIDEOID_PATTERN.matcher(content);
				/* 20130114 FIX
				group(3) が v で group(6) が id
				*/
				if (m.find() && id.equals(m.group(3)))
					return m.group(6);
			}
			try {
				String thumbinfo = GetThumbInfoUtil.get(id);
				if (thumbinfo != null) {
					Matcher m = THUMBINFO_VIDEOID_PATTERN.matcher(thumbinfo);
					if(m.find()) return m.group(1);
				}
				// コミュニティ動画はvideo.arrayから取得できる
				// 一方、一部のニコニコチャンネルの動画はvideo.arrayでは取得できず
				// getthumbinfoを使う必要がある(16スレ967)
				String xml = getPageContent(URL_VIDEOARRAY_API + id);
				if (xml != null) {
					Matcher m = XML_VIDEO_ID_PATTERN.matcher(xml);
					if (m.find()) return m.group(1);
				}
			} catch (IOException e) {}
		}
		return smid;
	}
	
	private String recognizeTitle(String smid, String content) {
		String title = NicoCachingTitleRetriever.getTitleFromResponse(content);
		if (title == null) {
			try {
				content = GetThumbInfoUtil.get(smid);
				title = NicoCachingTitleRetriever.getTitleFromXml(content);
			} catch (IOException e) {}
		}
		if (title != null) {
			// nl本体を通すと不要なRewriterを通るので自力でタイトルを設定する
			NicoCachingTitleRetriever.putTitleCache(smid, title);
		} else {
			return "(no title)";
		}
		util.debug("recognizeTitle: %s %s", smid, title);
		return title;
	}
	
	private WatchVars getWatchVars(String smid, String content) {
		// WatchVarsはrecognizeTitle内でキャッシュされているはず
		WatchVars wv = WatchVars.get(smid);
		if (wv == null) {
			wv = WatchVars.get(content);
		}
		return wv;
	}
	
	private String getFlvInfoFromWatchJson(JsonObject json) {
		if (json == null) {
			return null;
		}
		String flvInfo = json.getString("flashvars", "flvInfo");
		if (flvInfo == null) {
			return null;
		}
		try {
			return URLDecoder.decode(flvInfo, "UTF-8");
		} catch (UnsupportedEncodingException e) {
			return null;
		}
	}
	
	private String getFlvInfo(String content, JsonObject json) {
		String flvInfo = null;
		if (json != null) {
			flvInfo = getFlvInfoFromWatchJson(json);
		}
		if (flvInfo == null) {
			flvInfo = getflv(content);
		}
		if (flvInfo == null) {
			return null;
		}
		try { // nl本体にgetflvの内容を書き換えさせる(vid2cid対策)
			String url_param = "/" + fetchId;
			String getflvUrl = "http://flapi.nicovideo.jp/api/getflv" + url_param;
			HttpRequestHeader getflvRequestHeader =
					cloneHttpRequestHeader(fetchRequestHeader);
			ExtUtil.setURL(getflvRequestHeader, getflvUrl, false);
			flvInfo = Main.getRewriterProcessor().stringRewriter(
					getflvUrl, flvInfo, getflvRequestHeader, null);
		} catch (IOException e) {
			Logger.error(e);
		}
		debug2("flvInfo: %s\n", flvInfo);
		return flvInfo;
	}
	
	private static final Pattern GETFLV_URL_PATTERN = Pattern.compile(
			"(?<=^|&)url=(\\w+%3A%2F%2F[^&]+)(?=&|$)");
	private static final Pattern SM_FLV_PATTERN = Pattern.compile(
			"^http://(?!tn-skr)[^/]+(?:smilevideo|nicovideo)\\.jp/smile\\?(\\w)=([^.]+)\\.\\d+(as3)?(low)?$");
	private static final Pattern GETFLV_DELETED_PATTERN = Pattern.compile(
			"(?<=^|&)(deleted=\\d)(?=&|$)");
	
	private String getMovieUrlFromFlvInfo(String flvInfo) {
		Matcher m = GETFLV_DELETED_PATTERN.matcher(flvInfo);
		if (m.find()) {
			return m.group(1);	// 動画は削除済
		}
		m = GETFLV_URL_PATTERN.matcher(flvInfo);
		if (!m.find()) {
			util.info("cannot get movie_url.");
			util.debug("%s", flvInfo);
			return null;
		}
		try {
			return URLDecoder.decode(m.group(1), "UTF-8");
		} catch (UnsupportedEncodingException e) {
			Logger.error(e);
			return null;
		}
	}
	
	private VideoDescriptor determineVideoDescriptorFromSmileUrl(String smid, String movie_url) {
		Matcher m = SM_FLV_PATTERN.matcher(movie_url);
		if (!m.matches()) {
			util.info("unknown movie_url.");
			util.debug("%s", movie_url);
			return null;
		}
		String postfix = Cache.FLV;
		switch (m.group(1)) {
			case "v": postfix = Cache.FLV; break;
			case "m": postfix = Cache.MP4; break;
			case "s": postfix = Cache.SWF; break;
		}
		VideoDescriptor video = VideoDescriptor.newClassic(
				smid, postfix, m.group(4) != null);
		return video;
	}
	
	private String dmcApiUrl;
	private JsonObject dmcApiData;
	private HttpRequestHeader dmcApiRequestHeader = null;
	private HttpRequestHeader dmcRequestHeader = null;
	private volatile boolean heartbeatMode = false;
	
	private boolean isHearbeatThread() {
		return heartbeatMode;
	}
	private void runHeartbeat() {
		heartbeatMode = false;
		while (!Thread.interrupted()) {
			long sleepingTime = 30 * 1000;
			try {
				Thread.sleep(sleepingTime);
			} catch (InterruptedException e) {
				break;
			}
			dmcApiData = updateDmcSession(dmcApiData, dmcApiRequestHeader);
			if (dmcApiData == null) {
				util.warn("Failed to heartbeat.");
				return;
			}
		}
		deleteDmcSession(dmcApiData, dmcApiRequestHeader);
	}
	
	private Thread createDmcHeartbeatThread() {
		heartbeatMode = true;
		Thread thread = new Thread(this);
		return thread;
	}
	
	private void shutdownDmcHeartbeatThread(Thread thread) {
		thread.interrupt();
		try {
			thread.join();
		} catch (InterruptedException e) {
		}
	}
	
	private HttpRequestHeader cloneHttpRequestHeader(HttpRequestHeader requestHeader) {
		String s = requestHeader.toString();
		try {
			return new HttpRequestHeader(s);
		} catch (IOException e) {
			util.warn("*BUG* failed to clone requestHeader: " + s);
			return null;
		}
	}
	
	private static final Pattern URL_HOST_PATTERN = Pattern.compile("^https?://([^/]+)");
	
	private JsonObject postJson(String url, JsonObject postData, HttpRequestHeader requestHeader) {
		Matcher m = URL_HOST_PATTERN.matcher(url);
		util.debug("postJson %s", url);
		try {
			String postString = postData.toJson();
			debug2("postData: %s", postString);
			
			URLResource r = new URLResource(url);
			r.setProxyMyself(); // nl本体を経由して処理させる
			// ExtUtil.setURL(requestHeader, url, false);
			requestHeader.setURI(url);
			requestHeader.setMethod("POST");
			if (m.find()) {
				requestHeader.setMessageHeader("Host", m.group(1));
			}
			requestHeader.setMessageHeader("accept", "application/json");
			requestHeader.setMessageHeader("content-type", "application/json");
			requestHeader.setMessageHeader("Accept-Encoding", "gzip, deflate");
			HttpResponseHeader responseHeader = r.getResponseHeader(
					new ByteArrayInputStream(postString.getBytes()), requestHeader);
			if (responseHeader == null || responseHeader.getStatusCode() / 100 != 2) {
				return null;
			}
			util.debug("postJson: statusCode=%d", responseHeader.getStatusCode());
			byte[] body = r.getResponseBody();
			if (body == null) {
				return null;
			}
			String response = new String(body, "UTF-8");
			debug2("Response: %s", response);
			JsonObject responseJson = Json.parseObject(response);
			return responseJson;
		} catch (IOException e) {
			return null;
		}
	}
	
	private JsonObject createDmcSession(JsonObject dmcInfo, HttpRequestHeader requestHeader) {
		JsonObject session_api = dmcInfo.getObject("session_api");
		if (session_api == null) {
			return null;
		}
		
		JsonObject session = new JsonObject();
		session.put("recipe_id", session_api.get("recipe_id"));
		session.put("content_id", session_api.get("content_id"));
		session.put("content_type", new JsonString("movie"));
		session.put("content_src_id_sets", new JsonArray().add(new JsonObject()
				.put("content_src_ids", new JsonArray().add(new JsonObject()
						.put("src_id_to_mux", new JsonObject()
								.put("video_src_ids", session_api.get("videos"))
								.put("audio_src_ids", session_api.get("audios")))))));
		session.put("timing_constraint", new JsonString("unlimited"));
		session.put("keep_method", new JsonObject()
				.put("heartbeat", new JsonObject()
						.put("lifetime", session_api.get("heartbeat_lifetime"))));
		session.put("protocol", new JsonObject()
				.put("name", session_api.get("protocols", 0))
				.put("parameters", new JsonObject()
						.put("http_parameters", new JsonObject()
								.put("parameters", new JsonObject()
										.put("http_output_download_parameters",
												new JsonObject())))));
		session.put("content_uri", new JsonString(""));
		session.put("session_operation_auth", new JsonObject()
				.put("session_operation_auth_by_signature", new JsonObject()
						.put("token", session_api.get("token"))
						.put("signature", session_api.get("signature"))));
		session.put("content_auth", new JsonObject()
				.put("auth_type", new JsonString("ht2"))
				.put("max_content_count", new JsonNumber(10))
				.put("content_key_timeout", new JsonNumber(600000))
				.put("service_id", new JsonString("nicovideo"))
				.put("service_user_id", session_api.get("service_user_id")));
		session.put("client_info", new JsonObject()
				.put("player_id", session_api.get("player_id")));
		session.put("priority", session_api.get("priority"));
		JsonObject apiData = new JsonObject().put("session", session);
		
		String apiUrl = dmcApiUrl + "?_format=json";
		JsonObject responseJson = postJson(apiUrl, apiData, requestHeader);
		if (responseJson != null) {
			return responseJson.getObject("data");
		} else {
			return null;
		}
	}
	
	private JsonObject updateDmcSession(JsonObject apiData, HttpRequestHeader requestHeader) {
		String sessionId = apiData.getString("session", "id");
		if (sessionId == null) {
			return null;
		}
		String apiUrl = dmcApiUrl + "/" + sessionId + "?_format=json&_method=PUT";
		JsonObject responseJson = postJson(apiUrl, apiData, requestHeader);
		if (responseJson != null) {
			return responseJson.getObject("data");
		} else {
			return null;
		}
	}
	
	private boolean deleteDmcSession(JsonObject apiData, HttpRequestHeader requestHeader) {
		String sessionId = apiData.getString("session", "id");
		if (sessionId == null) {
			return false;
		}
		String apiUrl = dmcApiUrl + "/" + sessionId + "?_format=json&_method=DELETE";
		JsonObject responseJson = postJson(apiUrl, apiData, requestHeader);
		if (responseJson != null) {
			return true;
		} else {
			return false;
		}
	}
	
	private static final Pattern DMC_VIDEO_TYPE_PATTERN = Pattern.compile("^archive_(?:[^_]+)_(\\d+)kbps_([^_]+)$");
	private static final Pattern DMC_AUDIO_TYPE_PATTERN = Pattern.compile("^archive_(?:[^_]+)_(\\d+)kbps$");
	private static final Pattern DMC_CONTENT_URI_PATTERN = Pattern.compile(
			"^http://\\w+\\.dmc\\.nico(?::\\d+)?/vod/ht2_nicovideo/nicovideo-(\\w+?)(\\d+)_(\\w+)");
	
	private VideoDescriptor determineVideoDescriptorFromDmcSession(String smid, JsonObject apiData) {
		String video_src_id = apiData.getString("session", "content_src_id_sets",
				0, "content_src_ids", 0, "src_id_to_mux", "video_src_ids", 0);
		String audio_src_id = apiData.getString("session", "content_src_id_sets",
				0, "content_src_ids", 0, "src_id_to_mux", "audio_src_ids", 0);
		String content_uri = apiData.getString("session", "content_uri");
		
		Matcher m = DMC_CONTENT_URI_PATTERN.matcher(content_uri);
		if (!m.find()) {
			return null;
		}
		//String id = m.group(1);
		String srcId = m.group(3);
		
		Matcher mv = DMC_VIDEO_TYPE_PATTERN.matcher(video_src_id);
		if (!mv.matches()) {
			return null;
		}
		int videoBitrate = Integer.parseInt(mv.group(1));
		String videoMode = mv.group(2);
		
		Matcher ma = DMC_AUDIO_TYPE_PATTERN.matcher(audio_src_id);
		if (!ma.matches()) {
			return null;
		}
		int audioBitrate = Integer.parseInt(ma.group(1));
		
		boolean low;
		NicoIdInfoCache.Entry idInfo = NicoIdInfoCache.getInstance().get(smid.substring(2));
		Boolean videoEconomy = idInfo.getDmcVideoEconomy(video_src_id);
		Boolean audioEconomy = idInfo.getDmcAudioEconomy(audio_src_id);
		if (videoEconomy == null || audioEconomy == null) {
			low = false;
		} else {
			low = videoEconomy || audioEconomy;
		}
		
		return VideoDescriptor.newDmc(smid, Cache.MP4, low,
				videoMode, videoBitrate, audioBitrate, srcId);
	}
	private String getMovieUrlFromDmcApiData(JsonObject apiData) {
		String movie_url = apiData.getString("session", "content_uri");
		return movie_url;
	}
	
	
	
	private String fetch(String movie_url, HttpRequestHeader requestHeader) {
		if (!checkDiskFreeSpace()) {
			return STATUS_NO_DISK;
		}
		util.info(movie_url);
		OutputStream out = null;
		try {
			URLResource r = new URLResource(movie_url);
			r.setProxyMyself(); // nl本体を経由してキャッシュさせる
			HttpResponseHeader responseHeader = r.getResponseHeader(
					null, ExtUtil.setURL(requestHeader, movie_url, false));
			if (responseHeader != null) {
				debug2("requestHeader:\n%sresponseHeader: %s",
						requestHeader, responseHeader);
				int statusCode = responseHeader.getStatusCode();
				if (statusCode == 200) {
					if (util.isDebug() && fetchCount() == 1) {
						out = new FileOutputStream("FETCHED");
					} else {
						out = new NullOutputStream();
					}
					// selectTransferToで本体のみ転送するとonResponseHeader()が
					// 呼ばれないのでここで初期化する
					contentLength = responseHeader.getContentLength();
					transferedLength = 0L;
					r.addTransferListener(this);
					r.selectTransferTo(null, out, requestHeader, false);
					return movie_url;
				}
				if (statusCode == 404) return STATUS_NOT_FOUND;
			}
		} catch (IOException e) {
			Logger.error(e);
		} finally {
			CloseUtil.close(out);
		}
		return STATUS_FAILED;
	}
	
	private byte[] fetchVideoHeader(String movie_url, HttpRequestHeader requestHeader) {
		try {
			URLResource r = new URLResource(movie_url);
			requestHeader.setMessageHeader("Range", "bytes=0-511");
			HttpResponseHeader responseHeader = r.getResponseHeader(
					null, ExtUtil.setURL(requestHeader, movie_url, false));
			if (responseHeader != null) {
				debug2("requestHeader:\n%sresponseHeader: %s",
						requestHeader, responseHeader);
				int statusCode = responseHeader.getStatusCode();
				if (statusCode == 206) {
                    byte[] result = r.getResponseBody();
                    if (result.length != 512) {
                        return null;
                    } else {
                        return result;
                    }
				}
				if (statusCode == 404) return null;
			}
		} catch (IOException e) {
			Logger.error(e);
		} finally {
			requestHeader.removeMessageHeader("Range");
		}
		return null;
	}
	
	private static final Pattern USER_SESSION_PATTERN = Pattern.compile(
			"user_session=(user_session_(\\d+)_\\w+)");
	
	private void overwriteUserSession(HttpRequestHeader requestHeader) {
		if (user_session != null) {
			String cookie = requestHeader.getMessageHeader("Cookie");
			if (cookie != null && cookie.contains("user_session=")) {
				Matcher m = USER_SESSION_PATTERN.matcher(cookie);
				if (!m.find() || m.group(1).equals(user_session))
					return;
				cookie = cookie.replaceFirst("user_session_\\d+_\\w+", user_session);
				debugWithId(m.group(1).concat(" replaced."));
			} else {
				cookie = "user_session=".concat(user_session);
			}
			requestHeader.setMessageHeader("Cookie", cookie);
			debugWithId(user_session.concat(" used."));
		}
	}
	
	private static boolean captureUserSession(String content, String user_id) {
		if (content != null) {
			if (user_id.isEmpty()) {
				if (user_session != null && System.getProperty("mail") == null) {
					util.info(user_session.concat(" removed."));
					user_session = null;
				}
			} else {
				Matcher m = USER_SESSION_PATTERN.matcher(content);
				if (m.find() && !m.group(1).equals(user_session) &&
						(m.group(2).equals(user_id) || "*".equals(user_id))) {
					user_session = m.group(1);
					util.info(user_session.concat(" captured."));
				}
			}
		}
		return user_session != null;
	}
	
	// 同時ログインを防ぐためにstatic synchronizedメソッドにする
	private static final String LOGIN_URL = "https://secure.nicovideo.jp/secure/login?site=niconico";
	private static synchronized boolean login(HttpRequestHeader requestHeader) {
		user_session = null;
		String mail = System.getProperty("mail");
		String password = System.getProperty("password");
		if(mail != null && password != null) {
			String postString = "mail=" + mail + "&password=" + password;
			try {
				URLResource r = new URLResource(LOGIN_URL);
				ExtUtil.setURL(requestHeader, LOGIN_URL, false);
				requestHeader.removeMessageHeader("Cookie");
				requestHeader.setMethod("POST");
				HttpResponseHeader res = r.getResponseHeader(
						new ByteArrayInputStream(postString.getBytes()), requestHeader);
				// String cookie = res != null ? res.getMessageHeader("Set-Cookie") : null;
				String cookie = getUserSession(res);
				if (captureUserSession(cookie, "*")) {
					util.info(mail.concat(" login succeeded."));
					return true;
				}
			} catch (IOException e) {
				if(util.isDebug()) Logger.error(e);
				util.warn(mail.concat(" login failed."));
			}
		}
		return false;
	}
	
	private static String getUserSession(HttpResponseHeader response) {
		String cookie = null;
		if (response != null) {
			Matcher m = USER_SESSION_PATTERN.matcher(response.toString());
			if (m.find()) {
				cookie =m.group(0) + ";";
			}
		}
		return cookie;
	}
	
	private boolean avoidMultipleAccess () {
		try {
			if (this.canceled) {
				throw new InterruptedException();
			}
			access_sem.acquire();
		}
		catch (InterruptedException e) {
			debugWithId("interrupted.");
			return false;
		}
		
		try {
			multiple_access_timeout += access_wait; // 10 seconds per 1 access
			multiple_access_timeout -= (System.currentTimeMillis() - last_access_time);
			last_access_time = System.currentTimeMillis();
			if (multiple_access_timeout < 0) {
				multiple_access_timeout = 0;
			}
			if (multiple_access_timeout > 3 * access_wait) {
				debugWithId("sleep for " + access_wait / 1000 + " seconds.");
				setStatusMessage("too many requests. wait");
				Thread.sleep(access_wait);
			}
			else {
				// 複数同時アクセスを避ける
				// 0, 1, 2 sec
				Thread.sleep(multiple_access_timeout / 10);
			}
			return true;
		}
		catch (InterruptedException e) {
			debugWithId("interrupted.");
			multiple_access_timeout -= access_wait;
			return false;
		}
		finally {
			access_sem.release();
		}
	}
	
}
