Get out of my heart - Solve CTF Challenge
Every functions are decompiled by using IDA Free
Step 1: Map out the necessary functions written by the author.
int __fastcall __noreturn main(int argc, const char **argv, const char **envp)
{
float v3; // xmm0_4
float v4; // xmm0_4
float v5; // xmm0_4
double v6; // xmm0_8
double v7; // xmm0_8
__m128i v8; // xmm1
__m128i v9; // xmm1
__m128i v10; // xmm1
int v11; // eax
double v12; // [rsp+8h] [rbp-3EC8h]
float s[4000]; // [rsp+10h] [rbp-3EC0h] BYREF
int v14; // [rsp+3E90h] [rbp-40h]
int v15; // [rsp+3E94h] [rbp-3Ch]
int v16; // [rsp+3E98h] [rbp-38h]
float v17; // [rsp+3E9Ch] [rbp-34h]
float v18; // [rsp+3EA0h] [rbp-30h]
float v19; // [rsp+3EA4h] [rbp-2Ch]
float v20; // [rsp+3EA8h] [rbp-28h]
float v21; // [rsp+3EACh] [rbp-24h]
float v22; // [rsp+3EB0h] [rbp-20h]
float v23; // [rsp+3EB4h] [rbp-1Ch]
int m; // [rsp+3EB8h] [rbp-18h]
float k; // [rsp+3EBCh] [rbp-14h]
float j; // [rsp+3EC0h] [rbp-10h]
float i; // [rsp+3EC4h] [rbp-Ch]
float v28; // [rsp+3EC8h] [rbp-8h]
float v29; // [rsp+3ECCh] [rbp-4h]
printf("\x1B[2J\x1B[?25l");
v29 = 0.0;
while ( 1 )
{
memset(s, 0, sizeof(s));
v28 = 0.0;
v3 = cos(v29);
v23 = v3;
v4 = sin(v29);
v22 = v4;
for ( i = -0.5; i <= 0.5; i = i + 0.0099999998 )
{
v21 = 0.40000001;
for ( j = -0.5; j <= 0.5; j = j + 0.0099999998 )
{
v12 = (float)((float)-j * j);
v5 = fabs(j);
v6 = pow((float)(1.2 * i) - (v5 + v5) / 3.0, 2.0);
*(float *)&v6 = (float)(v21 * v21) + v12 - v6;
v20 = *(float *)&v6;
if ( *(float *)&v6 >= 0.0 )
{
v7 = sqrt(v20);
*(float *)&v7 = v7 / (float)(2.0 - i);
v20 = *(float *)&v7;
for ( k = -*(float *)&v7; v20 >= k; k = (float)(v20 / 6.0) + k )
{
v19 = (float)(j * v23) - (float)(k * v22);
v18 = (float)(k * v23) + (float)(j * v22);
v17 = (float)(v18 / 2.0) + 1.0;
v8 = (__m128i)LODWORD(v19);
*(float *)v8.m128i_i32 = (float)((float)((float)(v19 * v17) + 0.5) * 80.0) + 10.0;
v16 = lroundf(COERCE_FLOAT(_mm_cvtsi128_si32(v8)));
v9 = (__m128i)_mm_xor_ps((__m128)LODWORD(i), (__m128)0x80000000);
*(float *)v9.m128i_i32 = (float)((float)((float)(*(float *)v9.m128i_i32 * v17) + 0.5) * 39.0) + 2.0;
v15 = lroundf(COERCE_FLOAT(_mm_cvtsi128_si32(v9)));
v14 = 100 * v15 + v16;
if ( v18 >= s[v14] )
{
s[v14] = v18;
if ( v18 >= v28 )
v28 = v18;
}
}
}
}
}
printf("\x1B[H");
for ( m = 0; m <= 3999; ++m )
{
if ( m % 100 )
{
v10 = (__m128i)LODWORD(s[m]);
*(float *)v10.m128i_i32 = (float)(*(float *)v10.m128i_i32 / v28) * 13.0;
v11 = asc_21EB[lroundf(COERCE_FLOAT(_mm_cvtsi128_si32(v10)))];
}
else
{
v11 = 10;
}
putchar(v11);
}
v29 = v29 + 0.012;
usleep(0xBB8u);
}
}
void escape()
{
__int64 v0; // [rsp+0h] [rbp-160h] BYREF
__int64 v1; // [rsp+8h] [rbp-158h] BYREF
__int64 v2; // [rsp+10h] [rbp-150h] BYREF
size_t n; // [rsp+18h] [rbp-148h] BYREF
char s[264]; // [rsp+20h] [rbp-140h] BYREF
char *v5; // [rsp+128h] [rbp-38h]
void *v6; // [rsp+130h] [rbp-30h]
void *s2; // [rsp+138h] [rbp-28h]
void *ptr; // [rsp+140h] [rbp-20h]
const char *v9; // [rsp+148h] [rbp-18h]
const char *v10; // [rsp+150h] [rbp-10h]
const char *v11; // [rsp+158h] [rbp-8h]
v11 = "3b 5e 31 11 48 53 22 36 20 2b 0c 23 39 2b 1c 3f 2d 00 10 45 22 19 2a 02 22 07 08 0f 2c 51 13 22 21 56 1c 19 36 0"
"3 08 30 22 24 13 4b 33 15 3a 30 3b 52 67 04 2c 52 17 12";
v10 = "46 6c 61 67 7b 48 6f 77 5f 63 61 6e 5f 74 68 69 73 5f 62 65 5f 61 5f 66 6c 61 67 7d";
v9 = "31 04 08 14 0b 2d 1d 28 30 05 3e 1a 37 11 37 01 16 3e 10 11 28 09 36 15 1c 04 15 22";
printf("Enter key: ");
if ( fgets(s, 256, _bss_start) )
{
s[strcspn(s, "\r\n")] = 0;
n = 0LL;
v2 = 0LL;
ptr = (void *)hex_to_bytes(v10, &n);
s2 = (void *)hex_to_bytes(v9, &v2);
if ( n == v2 && n && (xor_with_key(ptr, n, s), !memcmp(ptr, s2, n)) )
{
free(ptr);
free(s2);
v1 = 0LL;
v6 = (void *)hex_to_bytes(v11, &v1);
xor_with_key(v6, v1, s);
*((_BYTE *)v6 + v1) = 0;
v0 = 0LL;
v5 = (char *)base64_decode_bytes(v6, &v0);
puts(v5);
free(v6);
free(v5);
}
else
{
free(ptr);
free(s2);
puts("Key is incorrent");
}
}
}
_BYTE *__fastcall hex_to_bytes(const char *a1, _QWORD *a2)
{
size_t v2; // rax
size_t v3; // rax
__int64 v4; // rax
int v6; // [rsp+18h] [rbp-28h]
int v7; // [rsp+1Ch] [rbp-24h]
_BYTE *v8; // [rsp+20h] [rbp-20h]
size_t v9; // [rsp+28h] [rbp-18h]
size_t v10; // [rsp+30h] [rbp-10h]
size_t v11; // [rsp+30h] [rbp-10h]
__int64 v12; // [rsp+38h] [rbp-8h]
v9 = strlen(a1);
v8 = malloc((v9 >> 1) + 1);
v12 = 0LL;
v10 = 0LL;
while ( v10 < v9 )
{
while ( v10 < v9 && ((*__ctype_b_loc())[(unsigned __int8)a1[v10]] & 0x2000) != 0 )
++v10;
if ( v10 >= v9 )
break;
v2 = v10;
v11 = v10 + 1;
v7 = hex_value((unsigned int)a1[v2]);
if ( v7 < 0 )
break;
while ( v11 < v9 && ((*__ctype_b_loc())[(unsigned __int8)a1[v11]] & 0x2000) != 0 )
++v11;
if ( v11 >= v9 )
break;
v3 = v11;
v10 = v11 + 1;
v6 = hex_value((unsigned int)a1[v3]);
if ( v6 < 0 )
break;
v4 = v12++;
v8[v4] = v6 | (16 * v7);
}
if ( a2 )
*a2 = v12;
return v8;
}
unsigned __int64 __fastcall xor_with_key(__int64 a1, unsigned __int64 a2, const char *a3)
{
unsigned __int64 result; // rax
size_t v5; // [rsp+20h] [rbp-10h]
unsigned __int64 i; // [rsp+28h] [rbp-8h]
v5 = strlen(a3);
for ( i = 0LL; ; ++i )
{
result = i;
if ( i >= a2 )
break;
*(_BYTE *)(a1 + i) ^= a3[i % v5];
}
return result;
}
_BYTE *__fastcall base64_decode_bytes(const char *a1, _QWORD *a2)
{
size_t v2; // rax
char v3; // al
char v4; // al
__int64 v5; // rax
__int64 v6; // rax
__int64 v7; // rax
char v9; // [rsp+14h] [rbp-2Ch]
unsigned __int8 v10; // [rsp+15h] [rbp-2Bh]
unsigned __int8 v11; // [rsp+16h] [rbp-2Ah]
char v12; // [rsp+17h] [rbp-29h]
_BYTE *v13; // [rsp+18h] [rbp-28h]
unsigned __int8 j; // [rsp+27h] [rbp-19h]
__int64 v15; // [rsp+28h] [rbp-18h]
__int64 i; // [rsp+30h] [rbp-10h]
char v17; // [rsp+3Fh] [rbp-1h]
v17 = 0;
v2 = strlen(a1);
v13 = malloc(((3 * v2) >> 2) + 1);
v15 = 0LL;
for ( i = 0LL; a1[i]; ++i )
{
if ( a1[i] == 61 )
{
v3 = v17++;
*(&v9 + v3) = 64;
}
else
{
for ( j = 0; j <= 0x3Fu && base46_map[j] != a1[i]; ++j )
;
v4 = v17++;
*(&v9 + v4) = j;
}
if ( v17 == 4 )
{
v5 = v15++;
v13[v5] = 4 * v9 + (v10 >> 4);
if ( v11 != 64 )
{
v6 = v15++;
v13[v6] = 16 * v10 + (v11 >> 2);
}
if ( v12 != 64 )
{
v7 = v15++;
v13[v7] = (v11 << 6) + v12;
}
v17 = 0;
}
}
v13[v15] = 0;
if ( a2 )
*a2 = v15;
return v13;
}
__int64 __fastcall hex_value(char a1)
{
if ( a1 > 47 && a1 <= 57 )
return (unsigned int)(a1 - 48);
if ( a1 > 96 && a1 <= 102 )
return (unsigned int)(a1 - 87);
if ( a1 <= 64 || a1 > 70 )
return 0xFFFFFFFFLL;
return (unsigned int)(a1 - 55);
}
Step 2: Analyze each funtions
Function: main
Base on the output of the binary. I can see that function main doesn't relate to anything to the flag I need to find. It just prints out a heart
!!*!****!=!;;
=*!#############*!=!=;:
=*###$$$$$$$$$$#$#$##***!;=; ..........
!*##$$$$$@@@@@@@$$$$$$$###**!!;~~,---------,---,,..
!###$$@@@@@@@@@@@@@@@$@$$$$##**!!=;::~:::::~~~~~~---,..
!##$$@@@@@@@@@@@@@@@$@@@@$$$$##**!=;;;;;;:;:::::~~~--,,.
=!#$$$@@@@@@@@@@@@@@@@@@@@@@$$$###**!!=:==:;;;;:::~~~--,..
;!#$$$@@@@@@@@@@@@@@@@@@@@@@@$$$###**!!===;==;;;;:::~~--,..
;!#$$$@@@@@@@@@@@@@@@@@@@@@@@@$$$$##***!!=====;;;;:::~~-,,.
=!##$$@@@@@@@@@@@@@@@@@@@@@@@@$*$####***!!!!===;~;;::~~-,,.
!##$@$@$@@@@@@@@@@@@@@@@$@@#@@$$$$###*!!!;!!:==;;;:-~~-,..
=!##$$$@@$@@@@@@@@@@@@@@@@#@@@$$$#####**!!!!===;;~::~~-,,.
!*##$@$@@$@@@@@@@@@@@@@$@@@@@$$$$####***!!!===;;;::~~-,.
=!*#$$$$@@@@@@@@$@@@@@@@@@@@@$$$$####***!!!===;~;:~~~-,.
=!*##$$$$@@@@@@@@@@$@@@@@@@$$$$$####***!!!===;;::~~-,.
=!*##$$$$$@@@@@@@@$$@@@@$$$$$$#####***!!!==;;;::~--,.
=**####$#$$$@$@@@@@#@$$$$$$$$#####***!!!::;;::~~-,.
=!**###$$$$$$$$$$$$$$$$$$$$*#!##****!!===;;::~-,.
;=!***####$##$$$#$$$$$$##*##!#!***!!===;;::~-,.
;==!**####*##$#$$$$#########****!!!==;;::~--.
;==!!****####*#########*****!!!!==;;::~-,,
~;==!*********##**********!!!!==;;::~-,.
:;===!!*!*******!*****!!!!!==;::~~-..
:;;==!=!!!=!!!!!!!!!!!====;;:~--.
~;;;=====!=!!!!==!=====;;:~-,
~::;;;============;;~::~-
-~::;;;;;;;;;;;;;;:~-,
--~~:::::::::::~--
,----~~~:~:-,
.-----
.
Function: escape
Seems like this function is the one holds the information about the flag and it uses the remaining functions I listed above hex_to_bytes, xor_with_key, base64_decode_bytes to obfuscate the flag.
v11 = "3b 5e 31 11 48 53 22 36 20 2b 0c 23 39 2b 1c 3f 2d 00 10 45 22 19 2a 02 22 07 08 0f 2c 51 13 22 21 56 1c 19 36 03 08 30 22 24 13 4b 33 15 3a 30 3b 52 67 04 2c 52 17 12";
v10 = "46 6c 61 67 7b 48 6f 77 5f 63 61 6e 5f 74 68 69 73 5f 62 65 5f 61 5f 66 6c 61 67 7d";
v9 = "31 04 08 14 0b 2d 1d 28 30 05 3e 1a 37 11 37 01 16 3e 10 11 28 09 36 15 1c 04 15 22";
I try convert these hex values to UTF-8 only v10 is not gibberish Flag{How_can_this_be_a_flag}.
printf("Enter key: ");
if ( fgets(s, 256, _bss_start) )
{
s[strcspn(s, "\r\n")] = 0;
Variable s is the key. I think we just need to find out what is the key then I will be able to deobfuscate variable v11 (I think this variable holds the flag)
n = 0LL;
v2 = 0LL;
ptr = hex_to_bytes(v10, &n);
s2 = hex_to_bytes(v9, &v2);
Convert v10 and v9 to bytes in order to be XOR with s. n and v2 is the length of v10 and v9
( n == v2 && n && (xor_with_key((__int64)ptr, n, s), !memcmp(ptr, s2, n)) )
In order the condition to be true, ptr XOR s = s2. As a result, to find the key I just need to ptr XOR s2 = s
The result of s is whisper_of_the_heartwhisper_
v6 = hex_to_bytes(v11, &v1);
xor_with_key((__int64)v6, v1, s);
Now, I use the key to XOR v11
Function: xor_with_key
Because I found out that the key length is 28 and v11 length is 56. So I need to analyze how the function xor_with_key works.
*(_BYTE *)(a1 + i) ^= a3[i % key_len];
Base on the logic above, I wrote a python script to mimic its logic.
for i in range(len(v11)):
XOR_result = ord(bytes.fromhex(v11[i])) ^ key[i % len(key)]
print(XOR_result,end='')
The output is: L6Xb86PiOMSWQNCWHab1UqCqRbzP[9zQQ3nFYeWDJAL#VtHDL:w\7eM
Function: base64_decode_bytes
Now let's check this base64 decoder map
for ( j = 0; j <= 0x3Fu && base46_map[j] != a1[i]; ++j )
I found out that this base64 decoder uses a different map. I checked the value of base46_map
base46_map db '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz+/'
But wait something seems wrong, the encoded flag I XOR earlier L6Xb86PiOMSWQNCWHab1UqCqRbzP[9zQQ3nFYeWDJAL#VtHDL:w\7eM
has some unmatched char compare to the base46_map.
Step 3: Finding the mistake
I looked through all of the steps I did earlier, I think the key whisper_of_the_heartwhisper_ is wrong. Because whisper_ repeated, so I delete that repeated part into whisper_of_the_heart
I wrote a new script to XOR v11 with key:
key = list("whisper_of_the_heart")
v11 = "3b 5e 31 11 48 53 22 36 20 2b 0c 23 39 2b 1c 3f 2d 00 10 45 22 19 2a 02 22 07 08 0f 2c 51 13 22 21 56 1c 19 36 03 08 30 22 24 13 4b 33 15 3a 30 3b 52 67 04 2c 52 17 12".split(" ")
for i in range(len(v11)):
XOR_result = ord(bytes.fromhex(v11[i])) ^ ord(key[i % len(key)])
print(chr(XOR_result),end='')
The output is: L6Xb86PiOMSWQNCWHab1UqCqRbzPC7LVI3CqSbzDULz8CpHoT48pD7Hz
Now this string of base64 is valid to the map of base46_map. I use cyberchef to decode it.
The flag is: FIA{C4n_Y0u_H34r_My_H34rtB34t}