spencer/omnisearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

those who commit

Browse Source
This commit is contained in:
frosty
2026-03-10 02:32:51 -04:00
parent 24cec7a350
commit a11bf8bb6c
19 changed files with 1537 additions and 1284 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Routes/ImageProxy.c
+42 -48
View File
@@ -16,9 +16,9 @@ typedef struct {
static int is_allowed_domain(const char *url) {
const char *protocol = strstr(url, "://");
if (!protocol) {
protocol = url;
protocol = url;
} else {
protocol += 3;
protocol += 3;
}
const char *path = strchr(protocol, '/');
@@ -26,49 +26,46 @@ static int is_allowed_domain(const char *url) {
char host[256] = {0};
if (host_len >= sizeof(host)) {
host_len = sizeof(host) - 1;
host_len = sizeof(host) - 1;
}
strncpy(host, protocol, host_len);
const char *allowed_domains[] = {
"mm.bing.net",
"th.bing.com",
NULL
};
const char *allowed_domains[] = {"mm.bing.net", "th.bing.com", NULL};
for (int i = 0; allowed_domains[i] != NULL; i++) {
size_t domain_len = strlen(allowed_domains[i]);
size_t host_str_len = strlen(host);
size_t domain_len = strlen(allowed_domains[i]);
size_t host_str_len = strlen(host);
if (host_str_len >= domain_len) {
const char *suffix = host + host_str_len - domain_len;
if (strcmp(suffix, allowed_domains[i]) == 0) {
return 1;
if (host_str_len >= domain_len) {
const char *suffix = host + host_str_len - domain_len;
if (strcmp(suffix, allowed_domains[i]) == 0) {
return 1;
}
}
}
}
return 0;
}
static size_t write_callback(void *contents, size_t size, size_t nmemb,
void *userp) {
void *userp) {
size_t realsize = size * nmemb;
MemoryBuffer *buf = (MemoryBuffer *)userp;
if (buf->size + realsize > MAX_IMAGE_SIZE) {
return 0;
return 0;
}
if (buf->size + realsize > buf->capacity) {
size_t new_capacity = buf->capacity * 2;
if (new_capacity < buf->size + realsize) {
new_capacity = buf->size + realsize;
}
char *new_data = realloc(buf->data, new_capacity);
if (!new_data) return 0;
buf->data = new_data;
buf->capacity = new_capacity;
size_t new_capacity = buf->capacity * 2;
if (new_capacity < buf->size + realsize) {
new_capacity = buf->size + realsize;
}
char *new_data = realloc(buf->data, new_capacity);
if (!new_data)
return 0;
buf->data = new_data;
buf->capacity = new_capacity;
}
memcpy(buf->data + buf->size, contents, realsize);
@@ -79,38 +76,34 @@ static size_t write_callback(void *contents, size_t size, size_t nmemb,
int image_proxy_handler(UrlParams *params) {
const char *url = NULL;
for (int i = 0; i < params->count; i++) {
if (strcmp(params->params[i].key, "url") == 0) {
url = params->params[i].value;
break;
}
if (strcmp(params->params[i].key, "url") == 0) {
url = params->params[i].value;
break;
}
}
if (!url || strlen(url) == 0) {
send_response("Missing 'url' parameter");
return 0;
send_response("Missing 'url' parameter");
return 0;
}
if (!is_allowed_domain(url)) {
send_response("Domain not allowed");
return 0;
send_response("Domain not allowed");
return 0;
}
CURL *curl = curl_easy_init();
if (!curl) {
send_response("Failed to initialize curl");
return 0;
send_response("Failed to initialize curl");
return 0;
}
MemoryBuffer buf = {
.data = malloc(8192),
.size = 0,
.capacity = 8192
};
MemoryBuffer buf = {.data = malloc(8192), .size = 0, .capacity = 8192};
if (!buf.data) {
curl_easy_cleanup(curl);
send_response("Memory allocation failed");
return 0;
curl_easy_cleanup(curl);
send_response("Memory allocation failed");
return 0;
}
curl_easy_setopt(curl, CURLOPT_URL, url);
@@ -130,18 +123,19 @@ int image_proxy_handler(UrlParams *params) {
char content_type[64] = {0};
if (content_type_ptr) {
strncpy(content_type, content_type_ptr, sizeof(content_type) - 1);
strncpy(content_type, content_type_ptr, sizeof(content_type) - 1);
}
curl_easy_cleanup(curl);
if (res != CURLE_OK || response_code != 200) {
free(buf.data);
send_response("Failed to fetch image");
return 0;
free(buf.data);
send_response("Failed to fetch image");
return 0;
}
const char *mime_type = strlen(content_type) > 0 ? content_type : "image/jpeg";
const char *mime_type =
strlen(content_type) > 0 ? content_type : "image/jpeg";
serve_data(buf.data, buf.size, mime_type);
free(buf.data);
src/Routes/Images.c
+188 -155
View File
@@ -1,7 +1,7 @@
#include "Images.h"
#include "../Utility/Unescape.h"
#include "../Proxy/Proxy.h"
#include "../Scraping/Scraping.h"
#include "../Utility/Unescape.h"
#include <curl/curl.h>
#include <libxml/HTMLparser.h>
@@ -17,12 +17,12 @@ struct MemoryBlock {
};
static size_t ImageWriteCallback(void *data, size_t size, size_t nmemb,
void *userp) {
void *userp) {
size_t realsize = size * nmemb;
struct MemoryBlock *mem = (struct MemoryBlock *)userp;
char *ptr = (char *)realloc(mem->response, mem->size + realsize + 1);
if (ptr == NULL) {
return 0;
return 0;
}
mem->response = ptr;
memcpy(&(mem->response[mem->size]), data, realsize);
@@ -35,30 +35,30 @@ static char *fetch_images_html(const char *url) {
CURL *curl_handle;
struct MemoryBlock chunk = {.response = malloc(1), .size = 0};
if (!chunk.response) {
return NULL;
return NULL;
}
curl_handle = curl_easy_init();
if (!curl_handle) {
free(chunk.response);
return NULL;
free(chunk.response);
return NULL;
}
curl_easy_setopt(curl_handle, CURLOPT_URL, url);
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, ImageWriteCallback);
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)&chunk);
curl_easy_setopt(
curl_handle, CURLOPT_USERAGENT,
"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko");
curl_handle, CURLOPT_USERAGENT,
"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko");
curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, 10L);
apply_proxy_settings(curl_handle);
CURLcode res = curl_easy_perform(curl_handle);
if (res != CURLE_OK) {
free(chunk.response);
curl_easy_cleanup(curl_handle);
return NULL;
free(chunk.response);
curl_easy_cleanup(curl_handle);
return NULL;
}
curl_easy_cleanup(curl_handle);
@@ -71,15 +71,16 @@ int images_handler(UrlParams *params) {
int page = 1;
if (params) {
for (int i = 0; i < params->count; i++) {
if (strcmp(params->params[i].key, "q") == 0) {
raw_query = params->params[i].value;
} else if (strcmp(params->params[i].key, "p") == 0) {
int parsed = atoi(params->params[i].value);
if (parsed > 1) page = parsed;
for (int i = 0; i < params->count; i++) {
if (strcmp(params->params[i].key, "q") == 0) {
raw_query = params->params[i].value;
} else if (strcmp(params->params[i].key, "p") == 0) {
int parsed = atoi(params->params[i].value);
if (parsed > 1)
page = parsed;
}
}
}
}
context_set(&ctx, "query", raw_query);
@@ -87,7 +88,7 @@ int images_handler(UrlParams *params) {
snprintf(page_str, sizeof(page_str), "%d", page);
snprintf(prev_str, sizeof(prev_str), "%d", page > 1 ? page - 1 : 0);
snprintf(next_str, sizeof(next_str), "%d", page + 1);
context_set(&ctx, "page", page_str);
context_set(&ctx, "page", page_str);
context_set(&ctx, "prev_page", prev_str);
context_set(&ctx, "next_page", next_str);
@@ -95,207 +96,239 @@ int images_handler(UrlParams *params) {
context_set(&ctx, "query", display_query);
if (!raw_query || strlen(raw_query) == 0) {
send_response("<h1>No query provided</h1>");
if (display_query) free(display_query);
free_context(&ctx);
return -1;
send_response("<h1>No query provided</h1>");
if (display_query)
free(display_query);
free_context(&ctx);
return -1;
}
CURL *tmp = curl_easy_init();
if (!tmp) {
send_response("<h1>Error initializing curl</h1>");
if (display_query) free(display_query);
free_context(&ctx);
return -1;
send_response("<h1>Error initializing curl</h1>");
if (display_query)
free(display_query);
free_context(&ctx);
return -1;
}
char *encoded_query = curl_easy_escape(tmp, raw_query, 0);
curl_easy_cleanup(tmp);
if (!encoded_query) {
send_response("<h1>Error encoding query</h1>");
if (display_query) free(display_query);
free_context(&ctx);
return -1;
send_response("<h1>Error encoding query</h1>");
if (display_query)
free(display_query);
free_context(&ctx);
return -1;
}
char url[1024];
int first = (page - 1) * 32 + 1;
snprintf(url, sizeof(url),
"https://www.bing.com/images/search?q=%s&first=%d", encoded_query, first);
snprintf(url, sizeof(url), "https://www.bing.com/images/search?q=%s&first=%d",
encoded_query, first);
char *html = fetch_images_html(url);
if (!html) {
send_response("<h1>Error fetching images</h1>");
free(encoded_query);
free(display_query);
free_context(&ctx);
return -1;
send_response("<h1>Error fetching images</h1>");
free(encoded_query);
free(display_query);
free_context(&ctx);
return -1;
}
htmlDocPtr doc = htmlReadMemory(html, (int)strlen(html), NULL, NULL,
HTML_PARSE_RECOVER | HTML_PARSE_NOERROR);
HTML_PARSE_RECOVER | HTML_PARSE_NOERROR);
if (!doc) {
free(html);
free(encoded_query);
free(display_query);
free_context(&ctx);
return -1;
free(html);
free(encoded_query);
free(display_query);
free_context(&ctx);
return -1;
}
xmlXPathContextPtr xpathCtx = xmlXPathNewContext(doc);
if (!xpathCtx) {
xmlFreeDoc(doc);
free(html);
free(encoded_query);
free(display_query);
free_context(&ctx);
return -1;
xmlFreeDoc(doc);
free(html);
free(encoded_query);
free(display_query);
free_context(&ctx);
return -1;
}
xmlXPathObjectPtr xpathObj =
xmlXPathEvalExpression((const xmlChar *)"//div[@class='item']", xpathCtx);
xmlXPathEvalExpression((const xmlChar *)"//div[@class='item']", xpathCtx);
int image_count = 0;
char ***image_matrix = NULL;
int *inner_counts = NULL;
if (xpathObj && xpathObj->nodesetval) {
int nodes = xpathObj->nodesetval->nodeNr;
int nodes = xpathObj->nodesetval->nodeNr;
int max_images = (nodes < 32) ? nodes : 32;
image_matrix = malloc(sizeof(char **) * max_images);
inner_counts = malloc(sizeof(int) * max_images);
int max_images = (nodes < 32) ? nodes : 32;
image_matrix = malloc(sizeof(char **) * max_images);
inner_counts = malloc(sizeof(int) * max_images);
for (int i = 0; i < nodes; i++) {
if (image_count >= 32) break;
for (int i = 0; i < nodes; i++) {
if (image_count >= 32)
break;
xmlNodePtr node = xpathObj->nodesetval->nodeTab[i];
xmlNodePtr img_node = NULL;
xmlNodePtr tit_node = NULL;
xmlNodePtr des_node = NULL;
xmlNodePtr thumb_link = NULL;
xmlNodePtr node = xpathObj->nodesetval->nodeTab[i];
xmlNodePtr img_node = NULL;
xmlNodePtr tit_node = NULL;
xmlNodePtr des_node = NULL;
xmlNodePtr thumb_link = NULL;
for (xmlNodePtr child = node->children; child; child = child->next) {
if (child->type != XML_ELEMENT_NODE) continue;
for (xmlNodePtr child = node->children; child; child = child->next) {
if (child->type != XML_ELEMENT_NODE)
continue;
if (xmlStrcmp(child->name, (const xmlChar *)"a") == 0) {
xmlChar *class = xmlGetProp(child, (const xmlChar *)"class");
if (class) {
if (xmlStrstr(class, (const xmlChar *)"thumb") != NULL) {
thumb_link = child;
for (xmlNodePtr thumb_child = child->children; thumb_child; thumb_child = thumb_child->next) {
if (xmlStrcmp(thumb_child->name, (const xmlChar *)"div") == 0) {
xmlChar *div_class = xmlGetProp(thumb_child, (const xmlChar *)"class");
if (div_class && xmlStrcmp(div_class, (const xmlChar *)"cico") == 0) {
for (xmlNodePtr cico_child = thumb_child->children; cico_child; cico_child = cico_child->next) {
if (xmlStrcmp(cico_child->name, (const xmlChar *)"img") == 0) {
img_node = cico_child;
break;
if (xmlStrcmp(child->name, (const xmlChar *)"a") == 0) {
xmlChar *class = xmlGetProp(child, (const xmlChar *)"class");
if (class) {
if (xmlStrstr(class, (const xmlChar *)"thumb") != NULL) {
thumb_link = child;
for (xmlNodePtr thumb_child = child->children; thumb_child;
thumb_child = thumb_child->next) {
if (xmlStrcmp(thumb_child->name, (const xmlChar *)"div") == 0) {
xmlChar *div_class =
xmlGetProp(thumb_child, (const xmlChar *)"class");
if (div_class &&
xmlStrcmp(div_class, (const xmlChar *)"cico") == 0) {
for (xmlNodePtr cico_child = thumb_child->children;
cico_child; cico_child = cico_child->next) {
if (xmlStrcmp(cico_child->name, (const xmlChar *)"img") ==
0) {
img_node = cico_child;
break;
}
}
}
if (div_class)
xmlFree(div_class);
}
}
} else if (xmlStrstr(class, (const xmlChar *)"tit") != NULL) {
tit_node = child;
}
xmlFree(class);
}
} else if (xmlStrcmp(child->name, (const xmlChar *)"div") == 0) {
xmlChar *class = xmlGetProp(child, (const xmlChar *)"class");
if (class && xmlStrcmp(class, (const xmlChar *)"meta") == 0) {
for (xmlNodePtr meta_child = child->children; meta_child;
meta_child = meta_child->next) {
if (xmlStrcmp(meta_child->name, (const xmlChar *)"div") == 0) {
xmlChar *div_class =
xmlGetProp(meta_child, (const xmlChar *)"class");
if (div_class) {
if (xmlStrcmp(div_class, (const xmlChar *)"des") == 0) {
des_node = meta_child;
}
xmlFree(div_class);
}
} else if (xmlStrcmp(meta_child->name, (const xmlChar *)"a") ==
0) {
xmlChar *a_class =
xmlGetProp(meta_child, (const xmlChar *)"class");
if (a_class &&
xmlStrstr(a_class, (const xmlChar *)"tit") != NULL) {
tit_node = meta_child;
}
if (a_class)
xmlFree(a_class);
}
}
}
if (class)
xmlFree(class);
}
}
xmlChar *iurl =
img_node ? xmlGetProp(img_node, (const xmlChar *)"src") : NULL;
xmlChar *full_url =
thumb_link ? xmlGetProp(thumb_link, (const xmlChar *)"href") : NULL;
xmlChar *title = des_node
? xmlNodeGetContent(des_node)
: (tit_node ? xmlNodeGetContent(tit_node) : NULL);
xmlChar *rurl =
tit_node ? xmlGetProp(tit_node, (const xmlChar *)"href") : NULL;
if (iurl && strlen((char *)iurl) > 0) {
char *proxy_url = NULL;
CURL *esc_curl = curl_easy_init();
if (esc_curl) {
char *encoded = curl_easy_escape(esc_curl, (char *)iurl, 0);
if (encoded) {
size_t proxy_len = strlen("/proxy?url=") + strlen(encoded) + 1;
proxy_url = malloc(proxy_len);
if (proxy_url) {
snprintf(proxy_url, proxy_len, "/proxy?url=%s", encoded);
}
curl_free(encoded);
}
if (div_class) xmlFree(div_class);
curl_easy_cleanup(esc_curl);
}
}
} else if (xmlStrstr(class, (const xmlChar *)"tit") != NULL) {
tit_node = child;
}
xmlFree(class);
}
} else if (xmlStrcmp(child->name, (const xmlChar *)"div") == 0) {
xmlChar *class = xmlGetProp(child, (const xmlChar *)"class");
if (class && xmlStrcmp(class, (const xmlChar *)"meta") == 0) {
for (xmlNodePtr meta_child = child->children; meta_child; meta_child = meta_child->next) {
if (xmlStrcmp(meta_child->name, (const xmlChar *)"div") == 0) {
xmlChar *div_class = xmlGetProp(meta_child, (const xmlChar *)"class");
if (div_class) {
if (xmlStrcmp(div_class, (const xmlChar *)"des") == 0) {
des_node = meta_child;
}
xmlFree(div_class);
}
} else if (xmlStrcmp(meta_child->name, (const xmlChar *)"a") == 0) {
xmlChar *a_class = xmlGetProp(meta_child, (const xmlChar *)"class");
if (a_class && xmlStrstr(a_class, (const xmlChar *)"tit") != NULL) {
tit_node = meta_child;
}
if (a_class) xmlFree(a_class);
}
}
}
if (class) xmlFree(class);
}
}
xmlChar *iurl = img_node ? xmlGetProp(img_node, (const xmlChar *)"src") : NULL;
xmlChar *full_url = thumb_link ? xmlGetProp(thumb_link, (const xmlChar *)"href") : NULL;
xmlChar *title = des_node ? xmlNodeGetContent(des_node) : (tit_node ? xmlNodeGetContent(tit_node) : NULL);
xmlChar *rurl = tit_node ? xmlGetProp(tit_node, (const xmlChar *)"href") : NULL;
if (iurl && strlen((char *)iurl) > 0) {
char *proxy_url = NULL;
CURL *esc_curl = curl_easy_init();
if (esc_curl) {
char *encoded = curl_easy_escape(esc_curl, (char *)iurl, 0);
if (encoded) {
size_t proxy_len = strlen("/proxy?url=") + strlen(encoded) + 1;
proxy_url = malloc(proxy_len);
if (proxy_url) {
snprintf(proxy_url, proxy_len, "/proxy?url=%s", encoded);
image_matrix[image_count] = malloc(sizeof(char *) * 4);
image_matrix[image_count][0] =
proxy_url ? strdup(proxy_url) : strdup((char *)iurl);
image_matrix[image_count][1] = strdup(title ? (char *)title : "Image");
image_matrix[image_count][2] = strdup(rurl ? (char *)rurl : "#");
image_matrix[image_count][3] =
strdup(full_url ? (char *)full_url : "#");
inner_counts[image_count] = 4;
image_count++;
}
curl_free(encoded);
}
curl_easy_cleanup(esc_curl);
}
image_matrix[image_count] = malloc(sizeof(char *) * 4);
image_matrix[image_count][0] = proxy_url ? strdup(proxy_url) : strdup((char *)iurl);
image_matrix[image_count][1] = strdup(title ? (char *)title : "Image");
image_matrix[image_count][2] = strdup(rurl ? (char *)rurl : "#");
image_matrix[image_count][3] = strdup(full_url ? (char *)full_url : "#");
inner_counts[image_count] = 4;
image_count++;
if (iurl)
xmlFree(iurl);
if (title)
xmlFree(title);
if (rurl)
xmlFree(rurl);
if (full_url)
xmlFree(full_url);
}
if (iurl) xmlFree(iurl);
if (title) xmlFree(title);
if (rurl) xmlFree(rurl);
if (full_url) xmlFree(full_url);
}
}
context_set_array_of_arrays(&ctx, "images", image_matrix, image_count,
inner_counts);
inner_counts);
char *rendered = render_template("images.html", &ctx);
if (rendered) {
send_response(rendered);
free(rendered);
send_response(rendered);
free(rendered);
} else {
send_response("<h1>Error rendering image results</h1>");
send_response("<h1>Error rendering image results</h1>");
}
if (image_matrix) {
for (int i = 0; i < image_count; i++) {
for (int j = 0; j < 4; j++) {
free(image_matrix[i][j]);
for (int i = 0; i < image_count; i++) {
for (int j = 0; j < 4; j++) {
free(image_matrix[i][j]);
}
free(image_matrix[i]);
}
free(image_matrix[i]);
}
free(image_matrix);
free(image_matrix);
}
if (inner_counts) {
free(inner_counts);
free(inner_counts);
}
if (xpathObj) xmlXPathFreeObject(xpathObj);
if (xpathCtx) xmlXPathFreeContext(xpathCtx);
if (doc) xmlFreeDoc(doc);
if (xpathObj)
xmlXPathFreeObject(xpathObj);
if (xpathCtx)
xmlXPathFreeContext(xpathCtx);
if (doc)
xmlFreeDoc(doc);
free(html);
curl_free(encoded_query);
free(display_query);
free_context(&ctx);
return 0;
}
}
src/Routes/Search.c
+195 -172
View File
@@ -1,8 +1,8 @@
#include "Search.h"
#include "../Infobox/Wikipedia.h"
#include "../Infobox/Calculator.h"
#include "../Infobox/Dictionary.h"
#include "../Infobox/UnitConversion.h"
#include "../Infobox/Wikipedia.h"
#include "../Scraping/Scraping.h"
#include "../Utility/Display.h"
#include "../Utility/Unescape.h"
@@ -23,62 +23,66 @@ static void *wiki_thread_func(void *arg) {
InfoBoxThreadData *data = (InfoBoxThreadData *)arg;
char *dynamic_url = construct_wiki_url(data->query);
if (dynamic_url) {
data->result = fetch_wiki_data(dynamic_url);
data->success =
(data->result.title != NULL && data->result.extract != NULL &&
strlen(data->result.extract) > 10);
free(dynamic_url);
data->result = fetch_wiki_data(dynamic_url);
data->success =
(data->result.title != NULL && data->result.extract != NULL &&
strlen(data->result.extract) > 10);
free(dynamic_url);
} else {
data->success = 0;
data->success = 0;
}
return NULL;
}
static int is_calculator_query(const char *query) {
if (!query) return 0;
if (!query)
return 0;
int has_digit = 0;
int has_math_operator = 0;
for (const char *p = query; *p; p++) {
if (isdigit(*p) || *p == '.') {
has_digit = 1;
}
if (*p == '+' || *p == '-' || *p == '*' || *p == '/' || *p == '^') {
has_math_operator = 1;
}
if (isdigit(*p) || *p == '.') {
has_digit = 1;
}
if (*p == '+' || *p == '-' || *p == '*' || *p == '/' || *p == '^') {
has_math_operator = 1;
}
}
if (!has_digit || !has_math_operator) return 0;
if (!has_digit || !has_math_operator)
return 0;
int len = strlen(query);
for (int i = 0; i < len; i++) {
char c = query[i];
if (c == '+' || c == '-' || c == '*' || c == '/' || c == '^') {
int has_num_before = 0;
int has_num_after = 0;
char c = query[i];
if (c == '+' || c == '-' || c == '*' || c == '/' || c == '^') {
int has_num_before = 0;
int has_num_after = 0;
for (int j = i - 1; j >= 0; j--) {
if (isdigit(query[j]) || query[j] == '.') {
has_num_before = 1;
break;
}
if (query[j] != ' ') break;
}
for (int j = i - 1; j >= 0; j--) {
if (isdigit(query[j]) || query[j] == '.') {
has_num_before = 1;
break;
}
if (query[j] != ' ')
break;
}
for (int j = i + 1; j < len; j++) {
if (isdigit(query[j]) || query[j] == '.') {
has_num_after = 1;
break;
}
if (query[j] != ' ') break;
}
for (int j = i + 1; j < len; j++) {
if (isdigit(query[j]) || query[j] == '.') {
has_num_after = 1;
break;
}
if (query[j] != ' ')
break;
}
if (has_num_before || has_num_after) {
return 1;
if (has_num_before || has_num_after) {
return 1;
}
}
}
}
return 0;
}
@@ -87,11 +91,11 @@ static void *calc_thread_func(void *arg) {
InfoBoxThreadData *data = (InfoBoxThreadData *)arg;
if (is_calculator_query(data->query)) {
data->result = fetch_calc_data((char *)data->query);
data->success =
(data->result.title != NULL && data->result.extract != NULL);
data->result = fetch_calc_data((char *)data->query);
data->success =
(data->result.title != NULL && data->result.extract != NULL);
} else {
data->success = 0;
data->success = 0;
}
return NULL;
@@ -101,11 +105,11 @@ static void *dict_thread_func(void *arg) {
InfoBoxThreadData *data = (InfoBoxThreadData *)arg;
if (is_dictionary_query(data->query)) {
data->result = fetch_dictionary_data(data->query);
data->success =
(data->result.title != NULL && data->result.extract != NULL);
data->result = fetch_dictionary_data(data->query);
data->success =
(data->result.title != NULL && data->result.extract != NULL);
} else {
data->success = 0;
data->success = 0;
}
return NULL;
@@ -115,27 +119,30 @@ static void *unit_thread_func(void *arg) {
InfoBoxThreadData *data = (InfoBoxThreadData *)arg;
if (is_unit_conv_query(data->query)) {
data->result = fetch_unit_conv_data(data->query);
data->success =
(data->result.title != NULL && data->result.extract != NULL);
data->result = fetch_unit_conv_data(data->query);
data->success =
(data->result.title != NULL && data->result.extract != NULL);
} else {
data->success = 0;
data->success = 0;
}
return NULL;
}
static int add_infobox_to_collection(InfoBox *infobox, char ****collection,
int **inner_counts, int current_count) {
int **inner_counts, int current_count) {
*collection =
(char ***)realloc(*collection, sizeof(char **) * (current_count + 1));
(char ***)realloc(*collection, sizeof(char **) * (current_count + 1));
*inner_counts =
(int *)realloc(*inner_counts, sizeof(int) * (current_count + 1));
(int *)realloc(*inner_counts, sizeof(int) * (current_count + 1));
(*collection)[current_count] = (char **)malloc(sizeof(char *) * 4);
(*collection)[current_count][0] = infobox->title ? strdup(infobox->title) : NULL;
(*collection)[current_count][1] = infobox->thumbnail_url ? strdup(infobox->thumbnail_url) : NULL;
(*collection)[current_count][2] = infobox->extract ? strdup(infobox->extract) : NULL;
(*collection)[current_count][0] =
infobox->title ? strdup(infobox->title) : NULL;
(*collection)[current_count][1] =
infobox->thumbnail_url ? strdup(infobox->thumbnail_url) : NULL;
(*collection)[current_count][2] =
infobox->extract ? strdup(infobox->extract) : NULL;
(*collection)[current_count][3] = infobox->url ? strdup(infobox->url) : NULL;
(*inner_counts)[current_count] = 4;
@@ -148,15 +155,16 @@ int results_handler(UrlParams *params) {
int page = 1;
if (params) {
for (int i = 0; i < params->count; i++) {
if (strcmp(params->params[i].key, "q") == 0) {
raw_query = params->params[i].value;
} else if (strcmp(params->params[i].key, "p") == 0) {
int parsed = atoi(params->params[i].value);
if (parsed > 1) page = parsed;
for (int i = 0; i < params->count; i++) {
if (strcmp(params->params[i].key, "q") == 0) {
raw_query = params->params[i].value;
} else if (strcmp(params->params[i].key, "p") == 0) {
int parsed = atoi(params->params[i].value);
if (parsed > 1)
page = parsed;
}
}
}
}
context_set(&ctx, "query", raw_query);
@@ -164,14 +172,14 @@ int results_handler(UrlParams *params) {
snprintf(page_str, sizeof(page_str), "%d", page);
snprintf(prev_str, sizeof(prev_str), "%d", page > 1 ? page - 1 : 0);
snprintf(next_str, sizeof(next_str), "%d", page + 1);
context_set(&ctx, "page", page_str);
context_set(&ctx, "page", page_str);
context_set(&ctx, "prev_page", prev_str);
context_set(&ctx, "next_page", next_str);
if (!raw_query || strlen(raw_query) == 0) {
send_response("<h1>No query provided</h1>");
free_context(&ctx);
return -1;
send_response("<h1>No query provided</h1>");
free_context(&ctx);
return -1;
}
pthread_t wiki_tid, calc_tid, dict_tid, unit_tid;
@@ -181,36 +189,36 @@ int results_handler(UrlParams *params) {
InfoBoxThreadData unit_data = {.query = raw_query, .success = 0};
if (page == 1) {
pthread_create(&wiki_tid, NULL, wiki_thread_func, &wiki_data);
pthread_create(&calc_tid, NULL, calc_thread_func, &calc_data);
pthread_create(&dict_tid, NULL, dict_thread_func, &dict_data);
pthread_create(&unit_tid, NULL, unit_thread_func, &unit_data);
pthread_create(&wiki_tid, NULL, wiki_thread_func, &wiki_data);
pthread_create(&calc_tid, NULL, calc_thread_func, &calc_data);
pthread_create(&dict_tid, NULL, dict_thread_func, &dict_data);
pthread_create(&unit_tid, NULL, unit_thread_func, &unit_data);
}
ScrapeJob jobs[ENGINE_COUNT];
SearchResult *all_results[ENGINE_COUNT];
for (int i = 0; i < ENGINE_COUNT; i++) {
all_results[i] = NULL;
jobs[i].engine = &ENGINE_REGISTRY[i];
jobs[i].query = raw_query;
jobs[i].out_results = &all_results[i];
jobs[i].max_results = 10;
jobs[i].results_count = 0;
jobs[i].page = page;
jobs[i].handle = NULL;
jobs[i].response.memory = NULL;
jobs[i].response.size = 0;
jobs[i].response.capacity = 0;
all_results[i] = NULL;
jobs[i].engine = &ENGINE_REGISTRY[i];
jobs[i].query = raw_query;
jobs[i].out_results = &all_results[i];
jobs[i].max_results = 10;
jobs[i].results_count = 0;
jobs[i].page = page;
jobs[i].handle = NULL;
jobs[i].response.memory = NULL;
jobs[i].response.size = 0;
jobs[i].response.capacity = 0;
}
scrape_engines_parallel(jobs, ENGINE_COUNT);
if (page == 1) {
pthread_join(wiki_tid, NULL);
pthread_join(calc_tid, NULL);
pthread_join(dict_tid, NULL);
pthread_join(unit_tid, NULL);
pthread_join(wiki_tid, NULL);
pthread_join(calc_tid, NULL);
pthread_join(dict_tid, NULL);
pthread_join(unit_tid, NULL);
}
char ***infobox_matrix = NULL;
@@ -218,120 +226,135 @@ int results_handler(UrlParams *params) {
int infobox_count = 0;
if (page == 1) {
if (dict_data.success) {
infobox_count = add_infobox_to_collection(&dict_data.result, &infobox_matrix,
&infobox_inner_counts, infobox_count);
}
if (dict_data.success) {
infobox_count =
add_infobox_to_collection(&dict_data.result, &infobox_matrix,
&infobox_inner_counts, infobox_count);
}
if (calc_data.success) {
infobox_count = add_infobox_to_collection(&calc_data.result, &infobox_matrix,
&infobox_inner_counts, infobox_count);
}
if (calc_data.success) {
infobox_count =
add_infobox_to_collection(&calc_data.result, &infobox_matrix,
&infobox_inner_counts, infobox_count);
}
if (unit_data.success) {
infobox_count = add_infobox_to_collection(&unit_data.result, &infobox_matrix,
&infobox_inner_counts, infobox_count);
}
if (unit_data.success) {
infobox_count =
add_infobox_to_collection(&unit_data.result, &infobox_matrix,
&infobox_inner_counts, infobox_count);
}
if (wiki_data.success) {
infobox_count = add_infobox_to_collection(&wiki_data.result, &infobox_matrix,
&infobox_inner_counts, infobox_count);
}
if (wiki_data.success) {
infobox_count =
add_infobox_to_collection(&wiki_data.result, &infobox_matrix,
&infobox_inner_counts, infobox_count);
}
}
if (infobox_count > 0) {
context_set_array_of_arrays(&ctx, "infoboxes", infobox_matrix,
infobox_count, infobox_inner_counts);
for (int i = 0; i < infobox_count; i++) {
for (int j = 0; j < 4; j++) free(infobox_matrix[i][j]);
free(infobox_matrix[i]);
}
free(infobox_matrix);
free(infobox_inner_counts);
context_set_array_of_arrays(&ctx, "infoboxes", infobox_matrix,
infobox_count, infobox_inner_counts);
for (int i = 0; i < infobox_count; i++) {
for (int j = 0; j < 4; j++)
free(infobox_matrix[i][j]);
free(infobox_matrix[i]);
}
free(infobox_matrix);
free(infobox_inner_counts);
}
int total_results = 0;
for (int i = 0; i < ENGINE_COUNT; i++) {
total_results += jobs[i].results_count;
total_results += jobs[i].results_count;
}
if (total_results > 0) {
char ***results_matrix = (char ***)malloc(sizeof(char **) * total_results);
int *results_inner_counts = (int *)malloc(sizeof(int) * total_results);
char **seen_urls = (char **)malloc(sizeof(char *) * total_results);
int unique_count = 0;
char ***results_matrix = (char ***)malloc(sizeof(char **) * total_results);
int *results_inner_counts = (int *)malloc(sizeof(int) * total_results);
char **seen_urls = (char **)malloc(sizeof(char *) * total_results);
int unique_count = 0;
for (int i = 0; i < ENGINE_COUNT; i++) {
for (int j = 0; j < jobs[i].results_count; j++) {
char *display_url = all_results[i][j].url;
for (int i = 0; i < ENGINE_COUNT; i++) {
for (int j = 0; j < jobs[i].results_count; j++) {
char *display_url = all_results[i][j].url;
int is_duplicate = 0;
for (int k = 0; k < unique_count; k++) {
if (strcmp(seen_urls[k], display_url) == 0) {
is_duplicate = 1;
break;
int is_duplicate = 0;
for (int k = 0; k < unique_count; k++) {
if (strcmp(seen_urls[k], display_url) == 0) {
is_duplicate = 1;
break;
}
}
if (is_duplicate) {
free(all_results[i][j].url);
free(all_results[i][j].title);
free(all_results[i][j].snippet);
continue;
}
seen_urls[unique_count] = strdup(display_url);
results_matrix[unique_count] = (char **)malloc(sizeof(char *) * 4);
char *pretty_url = pretty_display_url(display_url);
results_matrix[unique_count][0] = strdup(display_url);
results_matrix[unique_count][1] = strdup(pretty_url);
results_matrix[unique_count][2] = all_results[i][j].title
? strdup(all_results[i][j].title)
: strdup("Untitled");
results_matrix[unique_count][3] =
all_results[i][j].snippet ? strdup(all_results[i][j].snippet)
: strdup("");
results_inner_counts[unique_count] = 4;
free(pretty_url);
free(all_results[i][j].url);
free(all_results[i][j].title);
free(all_results[i][j].snippet);
unique_count++;
}
free(all_results[i]);
}
if (is_duplicate) {
free(all_results[i][j].url);
free(all_results[i][j].title);
free(all_results[i][j].snippet);
continue;
context_set_array_of_arrays(&ctx, "results", results_matrix, unique_count,
results_inner_counts);
char *html = render_template("results.html", &ctx);
if (html) {
send_response(html);
free(html);
}
seen_urls[unique_count] = strdup(display_url);
results_matrix[unique_count] = (char **)malloc(sizeof(char *) * 4);
char *pretty_url = pretty_display_url(display_url);
results_matrix[unique_count][0] = strdup(display_url);
results_matrix[unique_count][1] = strdup(pretty_url);
results_matrix[unique_count][2] = all_results[i][j].title ? strdup(all_results[i][j].title) : strdup("Untitled");
results_matrix[unique_count][3] = all_results[i][j].snippet ? strdup(all_results[i][j].snippet) : strdup("");
results_inner_counts[unique_count] = 4;
free(pretty_url);
free(all_results[i][j].url);
free(all_results[i][j].title);
free(all_results[i][j].snippet);
unique_count++;
for (int i = 0; i < unique_count; i++) {
for (int j = 0; j < 4; j++)
free(results_matrix[i][j]);
free(results_matrix[i]);
free(seen_urls[i]);
}
free(all_results[i]);
}
context_set_array_of_arrays(&ctx, "results", results_matrix, unique_count, results_inner_counts);
char *html = render_template("results.html", &ctx);
if (html) {
send_response(html);
free(html);
}
for (int i = 0; i < unique_count; i++) {
for (int j = 0; j < 4; j++) free(results_matrix[i][j]);
free(results_matrix[i]);
free(seen_urls[i]);
}
free(seen_urls);
free(results_matrix);
free(results_inner_counts);
free(seen_urls);
free(results_matrix);
free(results_inner_counts);
} else {
char *html = render_template("results.html", &ctx);
if (html) {
send_response(html);
free(html);
}
char *html = render_template("results.html", &ctx);
if (html) {
send_response(html);
free(html);
}
}
if (page == 1) {
if (wiki_data.success) free_infobox(&wiki_data.result);
if (calc_data.success) free_infobox(&calc_data.result);
if (dict_data.success) free_infobox(&dict_data.result);
if (unit_data.success) free_infobox(&unit_data.result);
if (wiki_data.success)
free_infobox(&wiki_data.result);
if (calc_data.success)
free_infobox(&calc_data.result);
if (dict_data.success)
free_infobox(&dict_data.result);
if (unit_data.success)
free_infobox(&unit_data.result);
}
free_context(&ctx);
return 0;
}
}