Login: Added Login System

Login info should be placed in a file with the following format:
email@example.com
password

CMakeLists.txt

@@ -5,19 +5,39 @@ set(CMAKE_CXX_STANDARD 17)
 
 
 IF(DROPOUT_DL_BUILD_ALL)
-    add_executable(dropout-dl-minimal src/episode.cpp src/season.cpp src/series.cpp src/main.cpp)
+	add_executable(dropout-dl-minimal
+		src/login.cpp
+		src/episode.cpp
+		src/season.cpp
+		src/series.cpp
+		src/util.cpp
+		src/main.cpp)
 
     target_link_libraries(dropout-dl-minimal curl)
 
 
-    add_executable(dropout-dl-sqlite src/episode.cpp src/season.cpp src/series.cpp src/main.cpp)
+	add_executable(dropout-dl-sqlite
+		src/cookie.cpp
+		src/login.cpp
+		src/episode.cpp
+		src/season.cpp
+		src/series.cpp
+		src/util.cpp
+		src/main.cpp)
 
     target_link_libraries(dropout-dl-sqlite curl sqlite3)
 
     target_compile_definitions(dropout-dl-sqlite PUBLIC DROPOUT_DL_SQLITE)
 
 
-    add_executable(dropout-dl-full src/episode.cpp src/season.cpp src/series.cpp src/main.cpp)
+	add_executable(dropout-dl-full
+		src/cookie.cpp
+		src/login.cpp
+		src/episode.cpp
+		src/season.cpp
+		src/series.cpp
+		src/util.cpp
+		src/main.cpp)
 
     target_link_libraries(dropout-dl-full curl gcrypt sqlite3)
 
@@ -27,7 +47,14 @@ IF(DROPOUT_DL_BUILD_ALL)
     add_subdirectory(tests)
 ELSE()
 
-add_executable(dropout-dl src/episode.cpp src/season.cpp src/series.cpp src/main.cpp)
+add_executable(dropout-dl
+	src/cookie.cpp
+	src/login.cpp
+	src/episode.cpp
+	src/season.cpp
+	src/series.cpp
+	src/util.cpp
+	src/main.cpp)
 
 target_link_libraries(dropout-dl curl)
 

src/episode.cpp

@@ -187,26 +187,14 @@ namespace dropout_dl {
 
 		curl_easy_setopt(hnd, CURLOPT_WRITEFUNCTION, WriteCallback);
 		curl_easy_setopt(hnd, CURLOPT_WRITEDATA, &episode_data);
-		/* Here is a list of options the curl code used that cannot get generated
-		   as source easily. You may choose to either not use them or implement
-		   them yourself.
 
-		CURLOPT_WRITEDATA set to a objectpointer
+		std::string header;
-		CURLOPT_INTERLEAVEDATA set to a objectpointer
+		curl_easy_setopt(hnd, CURLOPT_HEADERFUNCTION, WriteCallback);
-		CURLOPT_WRITEFUNCTION set to a functionpointer
+		curl_easy_setopt(hnd, CURLOPT_HEADERDATA, &header);
-		CURLOPT_READDATA set to a objectpointer
-		CURLOPT_READFUNCTION set to a functionpointer
-		CURLOPT_SEEKDATA set to a objectpointer
-		CURLOPT_SEEKFUNCTION set to a functionpointer
-		CURLOPT_ERRORBUFFER set to a objectpointer
-		CURLOPT_STDERR set to a objectpointer
-		CURLOPT_HEADERFUNCTION set to a functionpointer
-		CURLOPT_HEADERDATA set to a objectpointer
-
-		*/
 
 		ret = curl_easy_perform(hnd);
 
+
 		curl_easy_cleanup(hnd);
 		hnd = nullptr;
 		curl_slist_free_all(slist1);
@@ -215,49 +203,6 @@ namespace dropout_dl {
 		return episode_data;
 	}
 
-	std::string get_generic_page(const std::string& url, bool verbose) {
-		CURL *hnd;
-		struct curl_slist *slist1;
-
-		std::string config_page;
-
-		slist1 = nullptr;
-		slist1 = curl_slist_append(slist1, "Accept: text/html");
-		slist1 = curl_slist_append(slist1, "Accept-Language: en-US,en");
-		slist1 = curl_slist_append(slist1, "Accept-Encoding: utf-8");
-		slist1 = curl_slist_append(slist1, "DNT: 1");
-		slist1 = curl_slist_append(slist1, "Connection: keep-alive");
-		slist1 = curl_slist_append(slist1, "Referer: https://www.dropout.tv/");
-		slist1 = curl_slist_append(slist1, "Upgrade-Insecure-Requests: 1");
-		slist1 = curl_slist_append(slist1, "Sec-Fetch-Dest: iframe");
-		slist1 = curl_slist_append(slist1, "Sec-Fetch-Mode: navigate");
-		slist1 = curl_slist_append(slist1, "Sec-Fetch-Site: cross-site");
-		slist1 = curl_slist_append(slist1, "Sec-GPC: 1");
-
-		hnd = curl_easy_init();
-		curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, 102400L);
-		curl_easy_setopt(hnd, CURLOPT_URL, url.c_str());
-		curl_easy_setopt(hnd, CURLOPT_NOPROGRESS, 1L);
-		curl_easy_setopt(hnd, CURLOPT_HTTPHEADER, slist1);
-		curl_easy_setopt(hnd, CURLOPT_MAXREDIRS, 50L);
-		curl_easy_setopt(hnd, CURLOPT_HTTP_VERSION, (long)CURL_HTTP_VERSION_2TLS);
-		curl_easy_setopt(hnd, CURLOPT_FTP_SKIP_PASV_IP, 1L);
-		curl_easy_setopt(hnd, CURLOPT_TCP_KEEPALIVE, 1L);
-		curl_easy_setopt(hnd, CURLOPT_VERBOSE, verbose);
-
-		curl_easy_setopt(hnd, CURLOPT_WRITEFUNCTION, WriteCallback);
-		curl_easy_setopt(hnd, CURLOPT_WRITEDATA, &config_page);
-
-		curl_easy_perform(hnd);
-
-		curl_easy_cleanup(hnd);
-		hnd = nullptr;
-		curl_slist_free_all(slist1);
-		slist1 = nullptr;
-
-		return config_page;
-	}
-
 	std::vector<std::string> episode::get_qualities() {
 		if (!qualities.empty()) {
 			return qualities;

src/episode.h

@@ -62,8 +62,8 @@ namespace dropout_dl {
 		/**
 		 *
 		 * @param url - The url of the episode page
-		 * @param auth_cookie - The authentication cookie with name "__cf_bm"
+		 * @param auth_cookie - The authentication cookie with name "__cf_bm".
-		 * @param session_cookie - The session cookie with name "_session"
+		 * @param session_cookie - The session cookie with name "_session".
 		 * @param verbose - Whether or not to be verbose (not recommended)
 		 * @return The episode page data
 		 */

src/login.cpp

@@ -0,0 +1,119 @@
+#include "login.h"
+
+
+void dropout_dl::login::get_cookies(std::string& session, std::string& cf_bm) {
+	std::string email;
+	std::string password;
+
+	std::cout << "Logging in...\n";
+
+	get_login_info_from_file("login", email, password);
+
+	/// Needed to login properly
+	std::string authentication;
+	get_login_tokens(session, cf_bm, authentication);
+
+	if (!login_with_tokens(email, password, session, cf_bm, authentication)) {
+		std::cerr << RED << "ERROR: Could not login. Check your login. If you are certain your information is correct please report this issue\n";
+		exit(1);
+	}
+
+	std::cout << GREEN << "Successfully logged in.\n" << RESET;
+}
+
+void dropout_dl::login::get_login_info_from_file(const std::string& filename, std::string& email, std::string& password) {
+	std::ifstream login_file(filename);
+
+	if (!login_file) {
+		std::cerr << "ERROR: Could not open login file\n";
+		exit(1);
+	}
+
+	std::getline(login_file, email);
+	std::getline(login_file, password);
+
+	if (email.empty() || password.empty()) {
+		std::cerr << "ERROR: Invalid login format in file. File must contain just your email then password on seperate lines. Example:\nemail@example.com\npassword123\n";
+		exit(1);
+	}
+
+	if (email.find("@") == std::string::npos || email.find(".") == std::string::npos) {
+		/// Not outputting email because that could potentially reveal password if they are in the opposite place.
+		std::cerr << "ERROR: Invalid email in login file\n";
+		exit(1);
+	}
+}
+
+void dropout_dl::login::get_login_tokens(std::string& session_token, std::string& cf_bm_token, std::string& authentication_token) {
+	std::string login_page_url = "https://www.dropout.tv/login";
+	std::string header_string = "";
+
+	std::string login_page_data = get_generic_page(login_page_url, false, &header_string);
+
+
+	session_token = get_substring_in(header_string, "set-cookie: _session=", ";");
+
+	cf_bm_token = get_substring_in(header_string, "set-cookie: __cf_bm=", ";");
+
+	authentication_token = get_substring_in(login_page_data, "<meta name=\"csrf-param\" content=\"authenticity_token\" />\n<meta name=\"csrf-token\" content=\"", "\"");
+}
+
+bool dropout_dl::login::login_with_tokens(const std::string& email, const std::string& password, std::string& session, const std::string& cf_bm, const std::string& authentication_token) {
+	CURLcode ret;
+	CURL *hnd;
+	struct curl_slist *slist1;
+
+	slist1 = NULL;
+
+	std::string email_encoded = url_encode(email), password_encoded = url_encode(password), authentication_token_encoded = url_encode(authentication_token);
+
+	std::string cookies = "Cookie: locale_det=en; referrer_url=https%3A%2F%2Fwww.dropout.tv%2F; _session=" + session + "; __cf_bm=" + cf_bm;
+
+	slist1 = curl_slist_append(slist1, "Content-Type: application/x-www-form-urlencoded");
+	slist1 = curl_slist_append(slist1, "Origin: https://www.dropout.tv");
+	slist1 = curl_slist_append(slist1, "Connection: keep-alive");
+	slist1 = curl_slist_append(slist1, "Referer: https://www.dropout.tv/login");
+	slist1 = curl_slist_append(slist1, cookies.c_str());
+
+	hnd = curl_easy_init();
+	std::string postfields = "email=" + email_encoded + "&authenticity_token=" + authentication_token_encoded + "&password=" + password_encoded;
+	long http_response_code = 0;
+	curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, 102400L);
+	curl_easy_setopt(hnd, CURLOPT_URL, "https://www.dropout.tv/login");
+	curl_easy_setopt(hnd, CURLOPT_NOPROGRESS, 1L);
+	curl_easy_setopt(hnd, CURLOPT_POSTFIELDS, postfields.c_str());
+	curl_easy_setopt(hnd, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)postfields.size());
+	curl_easy_setopt(hnd, CURLOPT_HTTPHEADER, slist1);
+	curl_easy_setopt(hnd, CURLOPT_USERAGENT, "curl/7.87.0");
+	curl_easy_setopt(hnd, CURLOPT_MAXREDIRS, 50L);
+	curl_easy_setopt(hnd, CURLOPT_HTTP_VERSION, (long)CURL_HTTP_VERSION_2TLS);
+	curl_easy_setopt(hnd, CURLOPT_CUSTOMREQUEST, "POST");
+	curl_easy_setopt(hnd, CURLOPT_FTP_SKIP_PASV_IP, 1L);
+	curl_easy_setopt(hnd, CURLOPT_TCP_KEEPALIVE, 1L);
+
+	std::string header_string;
+	curl_easy_setopt(hnd, CURLOPT_HEADERFUNCTION, dropout_dl::WriteCallback);
+	curl_easy_setopt(hnd, CURLOPT_HEADERDATA, &header_string);
+
+	/// Hide output
+	/// TODO
+	FILE* nullvoid = fopen("/dev/null", "r");
+	curl_easy_setopt(hnd, CURLOPT_WRITEDATA, nullvoid);
+
+
+	ret = curl_easy_perform(hnd);
+
+	// set the session token to the new value in the response header.
+	session = get_substring_in(header_string, "set-cookie: _session=", ";");
+
+
+	curl_easy_getinfo(hnd, CURLINFO_RESPONSE_CODE, &http_response_code);
+
+	curl_easy_cleanup(hnd);
+	hnd = NULL;
+	curl_slist_free_all(slist1);
+	slist1 = NULL;
+
+	/// 302 is returned on sucessful login otherwise 200.
+	return http_response_code == 302;
+}

src/login.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <vector>
+#include <fstream>
+#include <chrono>
+#include <thread>
+
+#include <curl/curl.h>
+
+#include "cookie.h"
+#include "util.h"
+#include "color.h"
+
+namespace dropout_dl {
+	namespace login {
+		void get_cookies(std::string& session, std::string& cf_bm);
+
+		void get_login_info_from_file(const std::string& filename, std::string& email, std::string& password);
+
+		void get_login_tokens(std::string& session_token, std::string& cf_bm_token, std::string& authentication_token);
+
+		/**
+		 *
+		 * @param email
+		 * @param password
+		 * @param session - _session cookie. this changes with the response header to the login request.
+		 * @param cf_bm - __cf_bm cookie. this does not change.
+		 * @param authentication_token - an authentication token that is set on the login page and changes every time. I don't understand the purpose of this.
+		 * @return true on successful login. false otherwise.
+		 *
+		 * Login with the provided tokens and change session token.
+		 */
+		bool login_with_tokens(const std::string& email, const std::string& password, std::string& session, const std::string& cf_bm, const std::string& authentication_token);
+	}
+}

src/main.cpp

@@ -1,6 +1,7 @@
 #include <iostream>
 
 #include "series.h"
+#include "login.h"
 #include <regex>
 
 #ifdef DROPOUT_DL_SQLITE
@@ -17,7 +18,8 @@ namespace dropout_dl {
 
 		std::string url;
 		bool verbose = false;
-		bool cookies_forced = false;
+		bool force_cookies = false;
+		bool browser_cookies = false;
 		bool is_series = false;
 		bool is_season = false;
 		bool is_episode = false;
@@ -70,6 +72,9 @@ namespace dropout_dl {
 					}
 					quality = args[++i];
 				}
+				else if (arg == "browser-cookies") {
+					browser_cookies = true;
+				}
 				else if (arg == "force-cookies") {
 					if (i + 2 >= args.size()) {
 						std::cerr << "ARGUMENT PARSE ERROR: --force-cookies used with too few following arguments\n";
@@ -77,7 +82,7 @@ namespace dropout_dl {
 					}
 					cookies.emplace_back(args[++i]);
 					cookies.emplace_back(args[++i]);
-					cookies_forced = true;
+					force_cookies = true;
 				}
 				else if (arg == "output") {
 					if (i + 1 >= args.size()) {
@@ -112,6 +117,7 @@ namespace dropout_dl {
 								 "\t--output                 Set the output filename. Only works for single episode downloads\n"
 								 "\t--output-directory       Set the directory where files are output\n"
 								 "\t--verbose                Display debug information while running\n"
+								 "\t--browser-cookies        Use cookies from the browser placed in 'firefox_profile' or 'chrome_profile'\n"
 								 "\t--force-cookies          Interpret the next to arguments as authentication cookie and session cookie\n"
 								 "\t--series                 Interpret the url as a link to a series and download all episodes from all seasons\n"
 								 "\t--season                 Interpret the url as a link to a season and download all episodes from all seasons\n"
@@ -125,6 +131,12 @@ namespace dropout_dl {
 				output_directory = ".";
 			}
 
+			if (browser_cookies && force_cookies) {
+				std::cerr << "ARGUMENT PARSE ERROR: Cannot use browser cookies and forced cookies\n";
+				// Default to browser cookies.
+				force_cookies = false;
+			}
+
 			if ((is_season && is_series) || (is_season && is_episode) || (is_series && is_episode)) {
 				std::cerr << "ARGUMENT PARSE ERROR: Mulitple parse type arguments used\n";
 			}
@@ -309,7 +321,7 @@ std::vector<dropout_dl::cookie> get_cookies_from_chrome(const std::filesystem::p
  * Determines whether to get cookies from firefox or chrome. This function should not be run if cookies are forced using the `--force-cookies` option.
  * This function checks firefox first so if both firefox and chrome profiles are provided it will use firefox.
  */
-std::vector<dropout_dl::cookie> get_cookies(bool verbose = false) {
+std::vector<dropout_dl::cookie> get_cookies_from_browser(bool verbose = false) {
 
 	std::filesystem::path firefox_profile("firefox_profile");
 	std::filesystem::path chrome_profile("chrome_profile");
@@ -359,8 +371,14 @@ int main(int argc, char** argv) {
 		std::cout << "Got episode url: " << options.url << " from program arguments\n";
 	}
 
-	if (!options.cookies_forced) {
+	if (options.browser_cookies) {
-		options.cookies = get_cookies(options.verbose);
+		options.cookies = get_cookies_from_browser(options.verbose);
+	}
+	else if (!options.force_cookies) {
+		std::string session, cf_bm;
+		dropout_dl::login::get_cookies(session, cf_bm);
+
+		options.cookies = {{"__cf_bm", cf_bm}, {"_session", session}};
 	}
 
 	if (options.is_series) {

src/util.cpp

@@ -174,4 +174,163 @@ namespace dropout_dl {
 		((std::string*)userp)->append((char*)contents, size * nmemb);
 		return size * nmemb;
 	}
+
+
+	std::string get_generic_page(const std::string& url, bool verbose, std::string* header_string) {
+		CURL *hnd;
+		struct curl_slist *slist1;
+
+		std::string page_data;
+
+		slist1 = nullptr;
+		slist1 = curl_slist_append(slist1, "Accept: text/html");
+		slist1 = curl_slist_append(slist1, "Accept-Language: en-US,en");
+		slist1 = curl_slist_append(slist1, "Accept-Encoding: utf-8");
+		slist1 = curl_slist_append(slist1, "DNT: 1");
+		slist1 = curl_slist_append(slist1, "Connection: keep-alive");
+		slist1 = curl_slist_append(slist1, "Referer: https://www.dropout.tv/");
+		slist1 = curl_slist_append(slist1, "Upgrade-Insecure-Requests: 1");
+		slist1 = curl_slist_append(slist1, "Sec-Fetch-Dest: iframe");
+		slist1 = curl_slist_append(slist1, "Sec-Fetch-Mode: navigate");
+		slist1 = curl_slist_append(slist1, "Sec-Fetch-Site: cross-site");
+		slist1 = curl_slist_append(slist1, "Sec-GPC: 1");
+
+		hnd = curl_easy_init();
+		curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, 102400L);
+		curl_easy_setopt(hnd, CURLOPT_URL, url.c_str());
+		curl_easy_setopt(hnd, CURLOPT_NOPROGRESS, 1L);
+		curl_easy_setopt(hnd, CURLOPT_HTTPHEADER, slist1);
+		curl_easy_setopt(hnd, CURLOPT_MAXREDIRS, 50L);
+		curl_easy_setopt(hnd, CURLOPT_HTTP_VERSION, (long)CURL_HTTP_VERSION_2TLS);
+		curl_easy_setopt(hnd, CURLOPT_FTP_SKIP_PASV_IP, 1L);
+		curl_easy_setopt(hnd, CURLOPT_TCP_KEEPALIVE, 1L);
+		curl_easy_setopt(hnd, CURLOPT_VERBOSE, verbose);
+
+		curl_easy_setopt(hnd, CURLOPT_WRITEFUNCTION, WriteCallback);
+		curl_easy_setopt(hnd, CURLOPT_WRITEDATA, &page_data);
+
+		if (header_string) {
+			curl_easy_setopt(hnd, CURLOPT_HEADERFUNCTION, WriteCallback);
+			curl_easy_setopt(hnd, CURLOPT_HEADERDATA, header_string);
+		}
+
+		curl_easy_perform(hnd);
+
+		curl_easy_cleanup(hnd);
+		hnd = nullptr;
+		curl_slist_free_all(slist1);
+		slist1 = nullptr;
+
+		return page_data;
+	}
+
+
+	std::string get_generic_page_with_cookies(const std::string& url, std::string& session, std::string& cf_bm) {
+		CURL *hnd;
+		struct curl_slist *slist1;
+
+		std::string page_data;
+
+		slist1 = nullptr;
+		std::string cookies = "Cookie: _session=" + session + "; __cf_bm=" + cf_bm;
+		slist1 = curl_slist_append(slist1, "Accept: text/html");
+		slist1 = curl_slist_append(slist1, "Accept-Language: en-US,en");
+		slist1 = curl_slist_append(slist1, "Accept-Encoding: utf-8");
+		slist1 = curl_slist_append(slist1, "DNT: 1");
+		slist1 = curl_slist_append(slist1, "Connection: keep-alive");
+		slist1 = curl_slist_append(slist1, "Referer: https://www.dropout.tv/");
+		slist1 = curl_slist_append(slist1, "Upgrade-Insecure-Requests: 1");
+		slist1 = curl_slist_append(slist1, "Sec-Fetch-Mode: navigate");
+		slist1 = curl_slist_append(slist1, "Sec-Fetch-Site: cross-site");
+		slist1 = curl_slist_append(slist1, cookies.c_str());
+		slist1 = curl_slist_append(slist1, "Sec-GPC: 1");
+
+		hnd = curl_easy_init();
+		curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, 102400L);
+		curl_easy_setopt(hnd, CURLOPT_URL, url.c_str());
+		curl_easy_setopt(hnd, CURLOPT_NOPROGRESS, 1L);
+		curl_easy_setopt(hnd, CURLOPT_HTTPHEADER, slist1);
+		curl_easy_setopt(hnd, CURLOPT_MAXREDIRS, 50L);
+		curl_easy_setopt(hnd, CURLOPT_HTTP_VERSION, (long)CURL_HTTP_VERSION_2TLS);
+		curl_easy_setopt(hnd, CURLOPT_FTP_SKIP_PASV_IP, 1L);
+		curl_easy_setopt(hnd, CURLOPT_TCP_KEEPALIVE, 1L);
+
+		curl_easy_setopt(hnd, CURLOPT_WRITEFUNCTION, WriteCallback);
+		curl_easy_setopt(hnd, CURLOPT_WRITEDATA, &page_data);
+
+		std::string header_string;
+		curl_easy_setopt(hnd, CURLOPT_HEADERFUNCTION, WriteCallback);
+		curl_easy_setopt(hnd, CURLOPT_HEADERDATA, &header_string);
+
+		curl_easy_perform(hnd);
+
+
+		if (header_string.find("set-cookie: _session=")) {
+			std::cout << "updated session " << session << "->";
+			session = get_substring_in(header_string, "set-cookie: _session=", ";");
+			std::cout << session << "\n";
+		}
+
+		curl_easy_cleanup(hnd);
+		hnd = nullptr;
+		curl_slist_free_all(slist1);
+		slist1 = nullptr;
+
+		return page_data;
+	}
+
+
+
+	std::string get_substring_in(const std::string& string, const std::string& begin, const std::string& end) {
+		size_t substring_start = string.find(begin);
+
+		if (substring_start == std::string::npos) {
+			std::cerr << "ERROR: Could not find start of substring\n";
+			return "";
+		}
+
+		// Skip over the contents of 'begin'
+		substring_start += begin.size();
+
+		size_t substring_end = string.find(end, substring_start);
+
+		if (substring_end == std::string::npos) {
+			std::cerr << "ERROR: Could not find end of substring\n";
+			return "";
+		}
+
+
+		return string.substr(substring_start, substring_end - substring_start);
+	}
+
+
+	// https://stackoverflow.com/questions/154536/encode-decode-urls-in-c
+	std::string url_encode(const std::string& value) {
+		static auto hex_digt = "0123456789ABCDEF";
+
+		std::string result;
+		result.reserve(value.size() << 1);
+
+		for (auto ch : value)
+			{
+				if ((ch >= '0' && ch <= '9')
+					|| (ch >= 'A' && ch <= 'Z')
+					|| (ch >= 'a' && ch <= 'z')
+					|| ch == '-' || ch == '_' || ch == '!'
+					|| ch == '\'' || ch == '(' || ch == ')'
+					|| ch == '*' || ch == '~' || ch == '.')  // !'()*-._~
+					{
+						result.push_back(ch);
+					}
+				else
+					{
+						result += std::string("%") +
+							hex_digt[static_cast<unsigned char>(ch) >> 4]
+							+  hex_digt[static_cast<unsigned char>(ch) & 15];
+					}
+			}
+
+		return result;
+	}
+
 }

src/util.h

@@ -1,6 +1,8 @@
 #pragma once
 
 #include <iostream>
+#include <sstream>
+#include <iomanip>
 
 #include <curl/curl.h>
 
@@ -107,14 +109,44 @@ namespace dropout_dl {
 	 */
 	size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp);
 
+
 	/**
 	 *
 	 * @param url - Url which is being downloaded
 	 * @param verbose - Whether or not to be verbose (not recommended)
+	 * @param header_string - A string to place header data in. optional
 	 * @return The page data as a string
 	 *
 	 * This function downloads the provided url and returns it as a string. Does not use cookies. This was ripped directly from a firefox network request for an episode page and modified minimally.
 	 */
-	std::string get_generic_page(const std::string& url, bool verbose = false);
+	std::string get_generic_page(const std::string& url, bool verbose = false, std::string* header_string = nullptr);
 
+	/**
+	 *
+	 * @param url - Url which is being downloaded
+	 * @param session - _session cookie. this is updated if possible
+	 * @param cf_bm - __cf_bm cookie. this is updated if possible
+	 * @return The page data as a string
+	 *
+	 * This function downloads the provided url and returns it as a string. Does not use cookies. This was ripped directly from a firefox network request for an episode page and modified minimally.
+	 */
+	std::string get_generic_page_with_cookies(const std::string& url, std::string& session, std::string& cf_bm);
+
+
+	/**
+	 *
+	 * @param string - the string that is searched
+	 * @param start - the starting string
+	 * @param end - the ending string
+	 * @return the substring of 'string' between 'start' and 'end'
+	 */
+	std::string get_substring_in(const std::string& string, const std::string& begin, const std::string& end);
+
+
+	/**
+	 *
+	 * @param value - the string to be encoded
+	 * @return 'value' with values escaped. e.g. "&" -> %26
+	 */
+	std::string url_encode(const std::string &value);
 }

tests/CMakeLists.txt

@@ -3,7 +3,14 @@ project(dropout-dl-tests)
 
 set(CMAKE_CXX_STANDARD 17)
 
-add_executable(test test.cpp episode_tests.cpp ../src/episode.cpp ../src/season.cpp series_tests.cpp ../src/series.cpp)
+add_executable(test
+	../src/util.cpp
+	test.cpp
+	episode_tests.cpp
+	../src/episode.cpp
+	../src/season.cpp
+	series_tests.cpp
+	../src/series.cpp)
 
 target_link_libraries(test curl sqlite3 gcrypt)
 

tests/test.h

@@ -6,33 +6,10 @@
 #include <vector>
 #include "iostream"
 
+#include "../src/color.h"
+
 namespace dropout_dl {
 
-#define RESET   "\033[0m"
-#define BLACK   "\033[30m"      /* Black */
-#define RED     "\033[31m"      /* Red */
-#define GREEN   "\033[32m"      /* Green */
-#define YELLOW  "\033[33m"      /* Yellow */
-#define BLUE    "\033[34m"      /* Blue */
-#define MAGENTA "\033[35m"      /* Magenta */
-#define CYAN    "\033[36m"      /* Cyan */
-#define WHITE   "\033[37m"      /* White */
-#define BOLDBLACK   "\033[1m\033[30m"      /* Bold Black */
-#define BOLDRED     "\033[1m\033[31m"      /* Bold Red */
-#define FAIL     "\033[31mFAIL: \033[0m"             // Test Failure
-#define BOLDFAIL     "\033[1m\033[31mFAIL: \033[0m"             // Test Failure
-#define BOLDGREEN   "\033[1m\033[32m"      /* Bold Green */
-#define SUCCESS   "\033[32mSUCCESS: \033[0m"         /* Test Success */
-#define BOLDSUCCESS   "\033[1m\033[32mSUCCESS: \033[0m"         /* Test Success */
-#define BOLDYELLOW  "\033[1m\033[33m"      /* Bold Yellow */
-#define WARN      "\033[1m\033[33mWARNING: \033[0m"         /* Test Warning */
-#define BOLDBLUE    "\033[1m\033[34m"      /* Bold Blue */
-#define TESTNAME    "\033[1m\033[34mTEST: \033[0m"      /* Bold Blue */
-#define BOLDMAGENTA "\033[1m\033[35m"      /* Bold Magenta */
-#define BOLDCYAN    "\033[1m\033[36m"      /* Bold Cyan */
-#define BOLDWHITE   "\033[1m\033[37m"      /* Bold White */
-
-
     template <typename t> class test {
         public:
             std::string name;