Commit c96bc62d authored by bol-van's avatar bol-van
Browse files

nfqws: ip->hostname cache

parent 8432388b

docs/changes.txt

+1 −1
Original line number Diff line number Diff line
@@ -488,5 +488,5 @@ nfqws,tpws: debug tls version, alpn, ech 
nfqws: --dpi-desync-fake-tls=! means default tls fake

nfqws: --dup*

nfqws: --orig*

nfqws: autottl cache

nfqws: ipcache of hop count and host names

init.d: remove --ipset parameter prohibition

nfq/conntrack.h

+1 −0
Original line number Diff line number Diff line
@@ -86,6 +86,7 @@ typedef struct 
	t_l7proto l7proto;

	bool l7proto_discovered;

	char *hostname;

	bool hostname_discovered;

	bool hostname_ah_check;			// should perform autohostlist checks

	

	t_reassemble reasm_orig;

nfq/desync.c

+99 −10
Original line number Diff line number Diff line
@@ -353,7 +353,7 @@ static void wssize_cutoff(t_ctrack *ctrack) 
}

static void forced_wssize_cutoff(t_ctrack *ctrack)

{

 	if (ctrack && ctrack->dp && ctrack->dp->wssize && !ctrack->b_wssize_cutoff)

	if (ctrack && ctrack->dp && !ctrack->b_wssize_cutoff)

	{

		DLOG("forced wssize-cutoff\n");

		wssize_cutoff(ctrack);
@@ -799,6 +799,49 @@ static void autottl_rediscover(t_ctrack *ctrack, const struct in_addr *a4, const 
	}

}



static bool ipcache_put_hostname(const struct in_addr *a4, const struct in6_addr *a6, const char *iface, const char *hostname)

{

	if (!params.cache_hostnames) return true;



	ip_cache_item *ipc = ipcacheTouch(&params.ipcache,a4,a6,iface);

	if (!ipc)

	{

		DLOG_ERR("ipcache_put_hostname: out of memory\n");

		return false;

	}

	free(ipc->hostname);

	if (!(ipc->hostname = strdup(hostname)))

	{

		DLOG_ERR("ipcache_put_hostname: out of memory\n");

		return false;

	}

	DLOG("hostname cached: %s\n", hostname);

	return true;

}

static bool ipcache_get_hostname(const struct in_addr *a4, const struct in6_addr *a6, const char *iface, char *hostname, size_t hostname_buf_len)

{

	if (!params.cache_hostnames)

	{

		*hostname = 0;

		return true;

	}

	ip_cache_item *ipc = ipcacheTouch(&params.ipcache,a4,a6,iface);

	if (!ipc)

	{

		DLOG_ERR("ipcache_get_hostname: out of memory\n");

		return false;

	}

	if (ipc->hostname)

	{

		DLOG("got cached hostname: %s\n", ipc->hostname);

		snprintf(hostname,hostname_buf_len,"%s",ipc->hostname);

	}

	else

		*hostname = 0;

	return true;

}





#ifdef BSD

// BSD pass to divert socket ip_id=0 and does not auto set it if sent via divert socket

static uint16_t IP4_IP_ID_FIX(const struct ip *ip)
@@ -1081,6 +1124,7 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint 
	uint32_t *timestamps;

	bool bSack,DF;

	uint16_t nmss;

	char host[256];



	uint32_t desync_fwmark = fwmark | params.desync_fwmark;

	extract_endpoints(dis->ip, dis->ip6, dis->tcp, NULL, &src, &dst);
@@ -1098,6 +1142,12 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint 
			DLOG("using cached desync profile %d\n",dp->n);

		else if (!ctrack_replay->dp_search_complete)

		{

			if (!ctrack_replay->hostname && !bReverse)

			{

				if (ipcache_get_hostname(dis->ip ? &dis->ip->ip_dst : NULL,dis->ip6 ? &dis->ip6->ip6_dst : NULL , ifout, host, sizeof(host)) && *host)

					if (!(ctrack_replay->hostname = strdup(host)))

						DLOG_ERR("strdup(host): out of memory\n");

			}

			dp = ctrack_replay->dp = dp_find(&params.desync_profiles, IPPROTO_TCP, (struct sockaddr *)&dst, ctrack_replay->hostname, ctrack_replay->l7proto, NULL, NULL, NULL);

			ctrack_replay->dp_search_complete = true;

		}
@@ -1121,7 +1171,18 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint 
			DLOG("using cached desync profile %d\n",dp->n);

		else if (!ctrack || !ctrack->dp_search_complete)

		{

			dp = dp_find(&params.desync_profiles, IPPROTO_TCP, (struct sockaddr *)&dst, ctrack ? ctrack->hostname : NULL, ctrack ? ctrack->l7proto : UNKNOWN, NULL, NULL, NULL);

			const char *hostname = NULL;

			if (ctrack)

			{

				hostname = ctrack->hostname;

				if (!hostname && !bReverse)

				{

					if (ipcache_get_hostname(dis->ip ? &dis->ip->ip_dst : NULL,dis->ip6 ? &dis->ip6->ip6_dst : NULL , ifout, host, sizeof(host)) && *host)

						if (!(hostname = ctrack_replay->hostname = strdup(host)))

							DLOG_ERR("strdup(host): out of memory\n");

				}

			}

			dp = dp_find(&params.desync_profiles, IPPROTO_TCP, (struct sockaddr *)&dst, hostname, ctrack ? ctrack->l7proto : UNKNOWN, NULL, NULL, NULL);

			if (ctrack)

			{

				ctrack->dp = dp;
@@ -1305,8 +1366,6 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint 
	{

		struct blob_collection_head *fake;



		char host[256];

		bool bHaveHost=false;

		uint8_t *p, *phost=NULL;

		const uint8_t *rdata_payload = dis->data_payload;

		size_t rlen_payload = dis->len_payload;
@@ -1315,6 +1374,7 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint 
		int multisplit_count;

		int i;

		uint16_t ip_id;

		bool bHaveHost=false;

		t_l7proto l7proto = UNKNOWN;



		if (replay)
@@ -1447,12 +1507,14 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint 
			bDiscoveredL7 = !ctrack_replay && l7proto!=UNKNOWN;

		if (bDiscoveredL7) DLOG("discovered l7 protocol\n");



		bool bDiscoveredHostname = bHaveHost && !(ctrack_replay && ctrack_replay->hostname);

		bool bDiscoveredHostname = bHaveHost && !(ctrack_replay && ctrack_replay->hostname_discovered);

		if (bDiscoveredHostname)

		{

			DLOG("discovered hostname\n");

			if (ctrack_replay)

			{

				ctrack_replay->hostname_discovered=true;

				free(ctrack_replay->hostname);

				ctrack_replay->hostname=strdup(host);

				if (!ctrack_replay->hostname)

				{
@@ -1460,6 +1522,12 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint 
					reasm_orig_cancel(ctrack);

					goto send_orig;

				}

				if (!ipcache_put_hostname(dis->ip ? &dis->ip->ip_dst : NULL,dis->ip6 ? &dis->ip6->ip6_dst : NULL , ifout, host))

				{

					reasm_orig_cancel(ctrack);

					goto send_orig;

				}



			}

		}
@@ -2262,6 +2330,7 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint 
	size_t pkt1_len, pkt2_len;

	uint8_t ttl_orig,ttl_fake;

	bool DF;

	char host[256];

	t_l7proto l7proto = UNKNOWN;



	extract_endpoints(dis->ip, dis->ip6, NULL, dis->udp, &src, &dst);
@@ -2279,6 +2348,12 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint 
			DLOG("using cached desync profile %d\n",dp->n);

		else if (!ctrack_replay->dp_search_complete)

		{

			if (!ctrack_replay->hostname && !bReverse)

			{

				if (ipcache_get_hostname(dis->ip ? &dis->ip->ip_dst : NULL,dis->ip6 ? &dis->ip6->ip6_dst : NULL , ifout, host, sizeof(host)) && *host)

					if (!(ctrack_replay->hostname = strdup(host)))

						DLOG_ERR("strdup(host): out of memory\n");

			}

			dp = ctrack_replay->dp = dp_find(&params.desync_profiles, IPPROTO_UDP, (struct sockaddr *)&dst, ctrack_replay->hostname, ctrack_replay->l7proto, NULL, NULL, NULL);

			ctrack_replay->dp_search_complete = true;

		}
@@ -2305,7 +2380,18 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint 
			DLOG("using cached desync profile %d\n",dp->n);

		else if (!ctrack || !ctrack->dp_search_complete)

		{

			dp = dp_find(&params.desync_profiles, IPPROTO_UDP, (struct sockaddr *)&dst, ctrack ? ctrack->hostname : NULL, ctrack ? ctrack->l7proto : UNKNOWN, NULL, NULL, NULL);

			const char *hostname = NULL;

			if (ctrack)

			{

				hostname = ctrack->hostname;

				if (!hostname && !bReverse)

				{

					if (ipcache_get_hostname(dis->ip ? &dis->ip->ip_dst : NULL,dis->ip6 ? &dis->ip6->ip6_dst : NULL , ifout, host, sizeof(host)) && *host)

						if (!(hostname = ctrack_replay->hostname = strdup(host)))

							DLOG_ERR("strdup(host): out of memory\n");

				}

			}

			dp = dp_find(&params.desync_profiles, IPPROTO_UDP, (struct sockaddr *)&dst, hostname, ctrack ? ctrack->l7proto : UNKNOWN, NULL, NULL, NULL);

			if (ctrack)

			{

				ctrack->dp = dp;
@@ -2353,7 +2439,6 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint 
	if (dis->len_payload)

	{

		struct blob_collection_head *fake;

		char host[256];

		bool bHaveHost=false;

		uint16_t ip_id;
@@ -2553,18 +2638,22 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint 
			bDiscoveredL7 = !ctrack_replay && l7proto!=UNKNOWN;

		if (bDiscoveredL7) DLOG("discovered l7 protocol\n");



		bool bDiscoveredHostname = bHaveHost && !(ctrack_replay && ctrack_replay->hostname);

		bool bDiscoveredHostname = bHaveHost && !(ctrack_replay && ctrack_replay->hostname_discovered);

		if (bDiscoveredHostname)

		{

			DLOG("discovered hostname\n");

			if (ctrack_replay)

			{

				ctrack_replay->hostname_discovered=true;

				free(ctrack_replay->hostname);

				ctrack_replay->hostname=strdup(host);

				if (!ctrack_replay->hostname)

				{

					DLOG_ERR("hostname dup : out of memory");

					goto send_orig;

				}

				if (!ipcache_put_hostname(dis->ip ? &dis->ip->ip_dst : NULL,dis->ip6 ? &dis->ip6->ip6_dst : NULL , ifout, host))

					goto send_orig;

			}

		}
@@ -2907,7 +2996,7 @@ static uint8_t dpi_desync_packet_play(bool replay, size_t reasm_offset, uint32_t 
}

uint8_t dpi_desync_packet(uint32_t fwmark, const char *ifin, const char *ifout, uint8_t *data_pkt, size_t *len_pkt)

{

	ipcachePurgeRateLimited(&params.ipcache, params.autottl_cache_lifetime);

	ipcachePurgeRateLimited(&params.ipcache, params.ipcache_lifetime);

	return dpi_desync_packet_play(false, 0, fwmark, ifin, ifout, data_pkt, len_pkt);

}

nfq/nfqws.c

+17 −11
Original line number Diff line number Diff line
@@ -99,9 +99,9 @@ static void onusr2(int sig) 
		printf("\nDESYNC PROFILE %d\n",dpl->dp.n);

		HostFailPoolDump(dpl->dp.hostlist_auto_fail_counters);

	}

	if (params.autottl_present)

	if (params.autottl_present || params.cache_hostnames)

	{

		printf("\nAUTOTTL IP CACHE\n");

		printf("\nIPCACHE\n");

		ipcachePrint(&params.ipcache);

	}

	printf("\n");
@@ -1417,7 +1417,8 @@ static void exithelp(void) 
		" --bind-fix6\t\t\t\t\t; apply outgoing interface selection fix for generated ipv6 packets\n"

#endif

		" --ctrack-timeouts=S:E:F[:U]\t\t\t; internal conntrack timeouts for TCP SYN, ESTABLISHED, FIN stages, UDP timeout. default %u:%u:%u:%u\n"

		" --autottl-cache-lifetime=<int>\t\t\t; time in seconds to keep cached hop count (default %u)\n"

		" --ipcache-lifetime=<int>\t\t\t; time in seconds to keep cached hop count and domain name (default %u)\n"

		" --ipcache-hostnames=[0|1]\t\t\t; 1 or no argument enables ip->hostname caching\n"

#ifdef __CYGWIN__

		"\nWINDIVERT FILTER:\n"

		" --wf-iface=<int>[.<int>]\t\t\t; numeric network interface and subinterface indexes\n"
@@ -1521,7 +1522,7 @@ static void exithelp(void) 
		" --dpi-desync-start=[n|d|s]N\t\t\t; apply dpi desync only to packet numbers (n, default), data packet numbers (d), relative sequence (s) greater or equal than N\n"

		" --dpi-desync-cutoff=[n|d|s]N\t\t\t; apply dpi desync only to packet numbers (n, default), data packet numbers (d), relative sequence (s) less than N\n",

		CTRACK_T_SYN, CTRACK_T_EST, CTRACK_T_FIN, CTRACK_T_UDP,

		AUTOTTL_CACHE_LIFETIME,

		IPCACHE_LIFETIME,

		HOSTLIST_AUTO_FAIL_THRESHOLD_DEFAULT, HOSTLIST_AUTO_FAIL_TIME_DEFAULT, HOSTLIST_AUTO_RETRANS_THRESHOLD_DEFAULT,

		AUTOTTL_DEFAULT_ORIG_DELTA,AUTOTTL_DEFAULT_ORIG_MIN,AUTOTTL_DEFAULT_ORIG_MAX,

		AUTOTTL_DEFAULT_DUP_DELTA,AUTOTTL_DEFAULT_DUP_MIN,AUTOTTL_DEFAULT_DUP_MAX,
@@ -1619,7 +1620,8 @@ enum opt_indices { 
	IDX_WSSIZE,

	IDX_WSSIZE_CUTOFF,

	IDX_CTRACK_TIMEOUTS,

	IDX_AUTOTTL_CACHE_LIFETIME,

	IDX_IPCACHE_LIFETIME,

	IDX_IPCACHE_HOSTNAMES,

	IDX_HOSTCASE,

	IDX_HOSTSPELL,

	IDX_HOSTNOSPACE,
@@ -1737,7 +1739,8 @@ static const struct option long_options[] = { 
	[IDX_WSSIZE] = {"wssize", required_argument, 0, 0},

	[IDX_WSSIZE_CUTOFF] = {"wssize-cutoff", required_argument, 0, 0},

	[IDX_CTRACK_TIMEOUTS] = {"ctrack-timeouts", required_argument, 0, 0},

	[IDX_AUTOTTL_CACHE_LIFETIME] = {"autottl-cache-lifetime", required_argument, 0, 0},

	[IDX_IPCACHE_LIFETIME] = {"ipcache-lifetime", required_argument, 0, 0},

	[IDX_IPCACHE_HOSTNAMES] = {"ipcache-hostnames", optional_argument, 0, 0},

	[IDX_HOSTCASE] = {"hostcase", no_argument, 0, 0},

	[IDX_HOSTSPELL] = {"hostspell", required_argument, 0, 0},

	[IDX_HOSTNOSPACE] = {"hostnospace", no_argument, 0, 0},
@@ -1891,7 +1894,7 @@ int main(int argc, char **argv) 
	params.ctrack_t_est = CTRACK_T_EST;

	params.ctrack_t_fin = CTRACK_T_FIN;

	params.ctrack_t_udp = CTRACK_T_UDP;

	params.autottl_cache_lifetime = AUTOTTL_CACHE_LIFETIME;

	params.ipcache_lifetime = IPCACHE_LIFETIME;



	LIST_INIT(&params.hostlists);

	LIST_INIT(&params.ipsets);
@@ -2045,13 +2048,16 @@ int main(int argc, char **argv) 
				exit_clean(1);

			}

			break;

		case IDX_AUTOTTL_CACHE_LIFETIME:

			if (sscanf(optarg, "%u", &params.autottl_cache_lifetime)!=1)

		case IDX_IPCACHE_LIFETIME:

			if (sscanf(optarg, "%u", &params.ipcache_lifetime)!=1)

			{

				DLOG_ERR("invalid autottl-cache-lifetime value\n");

				DLOG_ERR("invalid ipcache-lifetime value\n");

				exit_clean(1);

			}

			break;

		case IDX_IPCACHE_HOSTNAMES:

			params.cache_hostnames = !optarg || !!atoi(optarg);

			break;

		case IDX_HOSTCASE:

			dp->hostcase = true;

			break;
@@ -2958,7 +2964,7 @@ int main(int argc, char **argv) 
	}



	DLOG("initializing conntrack with timeouts tcp=%u:%u:%u udp=%u\n", params.ctrack_t_syn, params.ctrack_t_est, params.ctrack_t_fin, params.ctrack_t_udp);

	if (params.autottl_present) DLOG("autottl cache lifetime %us\n", params.autottl_cache_lifetime);

	if (params.autottl_present || params.cache_hostnames) DLOG("ipcache lifetime %us\n", params.ipcache_lifetime);

	ConntrackPoolInit(&params.conntrack, 10, params.ctrack_t_syn, params.ctrack_t_est, params.ctrack_t_fin, params.ctrack_t_udp);



#ifdef __linux__

nfq/params.h

+4 −3
Original line number Diff line number Diff line
@@ -36,7 +36,8 @@ 
#define	HOSTLIST_AUTO_FAIL_TIME_DEFAULT 	60

#define	HOSTLIST_AUTO_RETRANS_THRESHOLD_DEFAULT	3



#define AUTOTTL_CACHE_LIFETIME		86400

#define IPCACHE_LIFETIME		7200



#define AUTOTTL_DEFAULT_DESYNC_DELTA	-1

#define AUTOTTL_DEFAULT_DESYNC_MIN	3

#define AUTOTTL_DEFAULT_DESYNC_MAX	20
@@ -197,8 +198,8 @@ struct params_s 
	unsigned int ctrack_t_syn, ctrack_t_est, ctrack_t_fin, ctrack_t_udp;

	t_conntrack conntrack;



	unsigned int autottl_cache_lifetime;

	bool autottl_present;

	unsigned int ipcache_lifetime;

	bool autottl_present,cache_hostnames;

	ip_cache ipcache;

};