Don't cast second argument of mem_printf_hook (%b) to size_t.

Also treat the given number as unsigned int.

Due to the printf hook registration the second argument of
mem_printf_hook (if called via printf etc.) is always of type int*.
Casting this to a size_t pointer and then dereferencing that as int does
not work on big endian machines if int is smaller than size_t (e.g. on ppc64).

In order to make this change work if the argument is of a type larger
than int, size_t for instance, the second argument for %b has to be casted
to (u_)int.

src/libstrongswan/chunk.c

@@ -668,7 +668,8 @@ int chunk_printf_hook(char *dst, size_t len, printf_hook_spec_t *spec,
 
 	if (!spec->hash)
 	{
-		const void *new_args[] = {&chunk->ptr, &chunk->len};
+		u_int chunk_len = chunk->len;
+		const void *new_args[] = {&chunk->ptr, &chunk_len};
 		return mem_printf_hook(dst, len, spec, new_args);
 	}
 

src/libstrongswan/plugins/fips_prf/fips_prf.c

@@ -127,14 +127,14 @@ METHOD(prf_t, get_bytes, void,
 	{
 		/* a. XVAL = (XKEY + XSEED j) mod 2^b */
 		add_mod(this->b, xkey, xseed, xval);
-		DBG3(DBG_LIB, "XVAL %b", xval, this->b);
+		DBG3(DBG_LIB, "XVAL %b", xval, (u_int)this->b);
 		/* b. wi = G(t, XVAL ) */
 		this->g(this, chunk_create(xval, this->b), &w[i * this->b]);
-		DBG3(DBG_LIB, "w[%d] %b", i, &w[i * this->b], this->b);
+		DBG3(DBG_LIB, "w[%d] %b", i, &w[i * this->b], (u_int)this->b);
 		/* c. XKEY = (1 + XKEY + wi) mod 2b */
 		add_mod(this->b, xkey, &w[i * this->b], sum);
 		add_mod(this->b, sum, one, xkey);
-		DBG3(DBG_LIB, "XKEY %b", xkey, this->b);
+		DBG3(DBG_LIB, "XKEY %b", xkey, (u_int)this->b);
 	}
 
 	/* 3.3 done already, mod q not used */

src/libstrongswan/utils.c

@@ -444,7 +444,7 @@ int mem_printf_hook(char *dst, size_t dstlen,
 					printf_hook_spec_t *spec, const void *const *args)
 {
 	char *bytes = *((void**)(args[0]));
-	int len = *((size_t*)(args[1]));
+	u_int len = *((int*)(args[1]));
 
 	char buffer[BYTES_PER_LINE * 3];
 	char ascii_buffer[BYTES_PER_LINE + 1];
@@ -455,7 +455,7 @@ int mem_printf_hook(char *dst, size_t dstlen,
 	int i = 0;
 	int written = 0;
 
-	written += print_in_hook(dst, dstlen, "=> %d bytes @ %p", len, bytes);
+	written += print_in_hook(dst, dstlen, "=> %u bytes @ %p", len, bytes);
 
 	while (bytes_pos < bytes_roof)
 	{

src/libstrongswan/utils.h

@@ -637,7 +637,7 @@ int time_delta_printf_hook(char *dst, size_t len, printf_hook_spec_t *spec,
  * printf hook for memory areas.
  *
  * Arguments are:
- *	u_char *ptr, int len
+ *	u_char *ptr, u_int len
  */
 int mem_printf_hook(char *dst, size_t len, printf_hook_spec_t *spec,
 					const void *const *args);