This crackme can be downloaded from here.

First overview

I first start the program and it tells me straight away that the program is unregistered. The assignment was to create a keyfile generator so this is natural.

I then open the program in IDA Pro and looks for file opening functions and find one reference to fopen from sub_401490.

sub_401490

.text:00401490 sub_401490      proc near               ; DATA XREF: .rdata:0040241Co
.text:00401490
.text:00401490 var_58          = byte ptr -58h
.text:00401490 var_34          = byte ptr -34h
.text:00401490
.text:00401490                 sub     esp, 58h
.text:00401493                 xor     eax, eax
.text:00401495                 push    ebx
.text:00401496                 push    esi
.text:00401497                 mov     ebx, ecx
.text:00401499                 push    edi
.text:0040149A                 mov     ecx, 8
.text:0040149F                 lea     edi, [esp+64h+var_58]
.text:004014A3                 rep stosd
.text:004014A5                 stosb
.text:004014A6                 mov     ecx, 0Ch
.text:004014AB                 xor     eax, eax
.text:004014AD                 lea     edi, [esp+64h+var_34]
.text:004014B1                 push    offset aR       ; "r"
.text:004014B6                 rep stosd
.text:004014B8                 push    offset aKey_cyc ; "Key.cyc"
.text:004014BD                 stosw
.text:004014BF                 call    ds:fopen
.text:004014C5                 mov     esi, eax
.text:004014C7                 add     esp, 8
.text:004014CA                 test    esi, esi
.text:004014CC                 jz      short loc_401520
.text:004014CE                 mov     edi, ds:fgets
.text:004014D4                 push    esi             ; FILE *
.text:004014D5                 lea     eax, [esp+68h+var_58]
.text:004014D9                 push    1Fh             ; int
.text:004014DB                 push    eax             ; char *
.text:004014DC                 call    edi ; fgets
.text:004014DE                 push    esi             ; FILE *
.text:004014DF                 lea     ecx, [esp+74h+var_34]
.text:004014E3                 push    2Dh             ; int
.text:004014E5                 push    ecx             ; char *
.text:004014E6                 call    edi ; fgets
.text:004014E8                 lea     edx, [esp+7Ch+var_34]
.text:004014EC                 lea     eax, [esp+7Ch+var_58]
.text:004014F0                 push    edx
.text:004014F1                 push    eax
.text:004014F2                 call    sub_401410
.text:004014F7                 add     esp, 20h
.text:004014FA                 test    eax, eax
.text:004014FC                 jz      short loc_401505
.text:004014FE                 push    offset aRegged  ; "Regged!!!"
.text:00401503                 jmp     short loc_40150A
.text:00401505 ; ---------------------------------------------------------------------------
.text:00401505
.text:00401505 loc_401505:                             ; CODE XREF: sub_401490+6Cj
.text:00401505                 push    offset aUnRegistered ; "Un Registered!!!"
.text:0040150A
.text:0040150A loc_40150A:                             ; CODE XREF: sub_401490+73j
.text:0040150A                 push    3E8h
.text:0040150F                 mov     ecx, ebx
.text:00401511                 call    ?SetDlgItemTextA@CWnd@@QAEXHPBD@Z ; CWnd::SetDlgItemTextA(int,char const *)
.text:00401516                 push    esi             ; FILE *
.text:00401517                 call    ds:fclose
.text:0040151D                 add     esp, 4
.text:00401520
.text:00401520 loc_401520:                             ; CODE XREF: sub_401490+3Cj
.text:00401520                 pop     edi
.text:00401521                 pop     esi
.text:00401522                 pop     ebx
.text:00401523                 add     esp, 58h
.text:00401526                 retn
.text:00401526 sub_401490      endp

On address 004014BF the file named Key.cyc is opened, and if it cannot, the program jumps to loc_401520.

If the file is successfully opened two lines are read using fgets and these two strings are passed to sub_401410 which has accepted the content if it returns different from zero.

sub_401410

.text:00401410 sub_401410      proc near               ; CODE XREF: sub_401490+62p
.text:00401410
.text:00401410 var_34          = dword ptr -34h
.text:00401410 var_16          = byte ptr -16h
.text:00401410 arg_0           = dword ptr  4
.text:00401410 arg_4           = dword ptr  8
.text:00401410
.text:00401410                 sub     esp, 34h
.text:00401413                 or      ecx, 0FFFFFFFFh
.text:00401416                 xor     eax, eax
.text:00401418                 push    ebx
.text:00401419                 push    ebp
.text:0040141A                 mov     ebp, [esp+3Ch+arg_0]
.text:0040141E                 push    esi
.text:0040141F                 push    edi
.text:00401420                 mov     edi, ebp
.text:00401422                 xor     ebx, ebx
.text:00401424                 repne scasb
.text:00401426                 not     ecx
.text:00401428                 dec     ecx
.text:00401429                 mov     esi, ecx
.text:0040142B                 dec     esi
.text:0040142C                 cmp     esi, 5
.text:0040142F                 mov     [esi+ebp], al
.text:00401432                 jl      short loc_401485
.text:00401434                 cmp     esi, 1Ch
.text:00401437                 jg      short loc_401485
.text:00401439                 mov     edx, [esp+44h+arg_4]
.text:0040143D                 lea     eax, [esp+44h+var_34]
.text:00401441                 push    eax
.text:00401442                 mov     edi, edx
.text:00401444                 or      ecx, 0FFFFFFFFh
.text:00401447                 xor     eax, eax
.text:00401449                 repne scasb
.text:0040144B                 not     ecx
.text:0040144D                 dec     ecx
.text:0040144E                 push    ecx
.text:0040144F                 push    edx
.text:00401450                 call    sub_401190
.text:00401455                 add     esp, 0Ch
.text:00401458                 mov     [esp+44h+var_16], bl
.text:0040145C
.text:0040145C loc_40145C:                             ; CODE XREF: sub_401410+63j
.text:0040145C                 mov     eax, ebx
.text:0040145E                 cdq
.text:0040145F                 idiv    esi
.text:00401461                 movsx   ecx, byte ptr [edx+ebp]
.text:00401465                 xor     edx, edx
.text:00401467                 mov     dl, byte ptr [esp+ebx+44h+var_34]
.text:0040146B                 cmp     edx, ecx
.text:0040146D                 jnz     short loc_401475
.text:0040146F                 inc     ebx
.text:00401470                 cmp     ebx, 1Eh
.text:00401473                 jl      short loc_40145C
.text:00401475
.text:00401475 loc_401475:                             ; CODE XREF: sub_401410+5Dj
.text:00401475                 xor     eax, eax
.text:00401477                 pop     edi
.text:00401478                 cmp     ebx, 1Eh
.text:0040147B                 pop     esi
.text:0040147C                 pop     ebp
.text:0040147D                 pop     ebx
.text:0040147E                 setz    al
.text:00401481                 add     esp, 34h
.text:00401484                 retn
.text:00401485 ; ---------------------------------------------------------------------------
.text:00401485
.text:00401485 loc_401485:                             ; CODE XREF: sub_401410+22j
.text:00401485                                         ; sub_401410+27j
.text:00401485                 pop     edi
.text:00401486                 pop     esi
.text:00401487                 pop     ebp
.text:00401488                 xor     eax, eax
.text:0040148A                 pop     ebx
.text:0040148B                 add     esp, 34h
.text:0040148E                 retn
.text:0040148E sub_401410      endp

On address 0040141A and 00401420 the address of the first parameter (first line in the file) is put into EDI and on address 00401424 the REPNE SCASB instruction is used to search for the end of the string (first 0 byte or rather first byte with the same value as the AL register, which has value 0). REPNE SCASB decrements ECX for each repetition and ECX was initialized to -1 so the next two instructions calculates the number of characters in the string (including the nul termination) and the result is put into the EDI register which is decremented by one yielding the number of characters excluding the nul termination. The result must be greater than or equal to five and less than or equal to 28 or the function returns with zero.

So the first line must be between five and 28 characters (both included).

From address 00401439 to 0040144D the length of the second string is calculated (including nul termination) and the result and a pointer to the string and the local variable var_34 is passed to sub_401190 which we so far gives the following prototype: void sub_401190(char * file_string, size_t len, void * arg).

We do not yet know what sub_401190 does but I guess that it does some processing on the input and puts the result into our local var_34 buffer. With that assumption I continue the analysis of sub_401410.

Last part of sub_401410 runs a loop which checks if var_34 contains the same as arg_0. So it looks like the second argument to sub_401410 must be a name and that the first argument is a serial number which can be calculated based on the name and that sub_401190 does this calculation.

sub_401190

.text:00401190 sub_401190      proc near               ; CODE XREF: sub_401410+40p
.text:00401190
.text:00401190 var_1           = dword ptr -1
.text:00401190 arg_0           = dword ptr  7
.text:00401190 arg_4           = dword ptr  0Bh
.text:00401190 arg_8           = dword ptr  0Fh
.text:00401190
.text:00401190                 push    ecx
.text:00401191                 mov     eax, [esp+1+arg_4]
.text:00401195                 push    ebx
.text:00401196                 push    esi
.text:00401197                 xor     ecx, ecx
.text:00401199                 xor     esi, esi
.text:0040119B                 xor     ebx, ebx
.text:0040119D                 test    eax, eax
.text:0040119F                 jle     short loc_401220
.text:004011A1                 push    ebp
.text:004011A2                 mov     ebp, [esp+0Dh+arg_0]
.text:004011A6                 push    edi
.text:004011A7                 mov     edi, [esp+11h+arg_8]
.text:004011AB
.text:004011AB loc_4011AB:                             ; CODE XREF: sub_401190+86j
.text:004011AB                 mov     al, [ebx+ebp]
.text:004011AE                 cmp     al, 2Bh
.text:004011B0                 mov     byte ptr [esp+11h+arg_0], al
.text:004011B4                 jb      short loc_4011D1
.text:004011B6                 cmp     al, 7Ah
.text:004011B8                 ja      short loc_4011D1
.text:004011BA                 mov     eax, [esp+11h+arg_0]
.text:004011BE                 and     eax, 0FFh
.text:004011C3                 mov     al, ds:byte_40227D[eax]
.text:004011C9                 test    al, al
.text:004011CB                 jz      short loc_4011D7
.text:004011CD                 cmp     al, 24h
.text:004011CF                 jnz     short loc_4011D5
.text:004011D1
.text:004011D1 loc_4011D1:                             ; CODE XREF: sub_401190+24j
.text:004011D1                                         ; sub_401190+28j
.text:004011D1                 xor     al, al
.text:004011D3                 jmp     short loc_4011D7
.text:004011D5 ; ---------------------------------------------------------------------------
.text:004011D5
.text:004011D5 loc_4011D5:                             ; CODE XREF: sub_401190+3Fj
.text:004011D5                 add     al, 0C3h
.text:004011D7
.text:004011D7 loc_4011D7:                             ; CODE XREF: sub_401190+3Bj
.text:004011D7                                         ; sub_401190+43j
.text:004011D7                 dec     al
.text:004011D9                 mov     byte ptr [esp+ecx+11h+var_1], al
.text:004011DD                 inc     ecx
.text:004011DE                 cmp     ecx, 4
.text:004011E1                 jnz     short loc_40120F
.text:004011E3                 lea     ecx, [esp+11h+arg_8]
.text:004011E7                 lea     edx, [esp+11h+var_1]
.text:004011EB                 push    ecx
.text:004011EC                 push    edx
.text:004011ED                 call    sub_401150
.text:004011F2                 mov     al, byte ptr [esp+19h+arg_8]
.text:004011F6                 mov     dl, byte ptr [esp+19h+arg_8+1]
.text:004011FA                 add     esp, 8
.text:004011FD                 xor     ecx, ecx
.text:004011FF                 mov     [esi+edi], al
.text:00401202                 mov     al, byte ptr [esp+11h+arg_8+2]
.text:00401206                 inc     esi
.text:00401207                 mov     [esi+edi], dl
.text:0040120A                 inc     esi
.text:0040120B                 mov     [esi+edi], al
.text:0040120E                 inc     esi
.text:0040120F
.text:0040120F loc_40120F:                             ; CODE XREF: sub_401190+51j
.text:0040120F                 mov     eax, [esp+11h+arg_4]
.text:00401213                 inc     ebx
.text:00401214                 cmp     ebx, eax
.text:00401216                 jl      short loc_4011AB
.text:00401218                 pop     edi
.text:00401219                 mov     eax, esi
.text:0040121B                 pop     ebp
.text:0040121C                 pop     esi
.text:0040121D                 pop     ebx
.text:0040121E                 pop     ecx
.text:0040121F                 retn
.text:00401220 ; ---------------------------------------------------------------------------
.text:00401220
.text:00401220 loc_401220:                             ; CODE XREF: sub_401190+Fj
.text:00401220                 mov     eax, esi
.text:00401222                 pop     esi
.text:00401223                 pop     ebx
.text:00401224                 pop     ecx
.text:00401225                 retn
.text:00401225 sub_401190      endp

sub_401190 is pretty long and on address 004011ED it calls on to sub_401150 which I will start with because it is short and gives us a clear indication on what happens.

.text:00401150 sub_401150      proc near               ; CODE XREF: sub_401190+5Dp
.text:00401150
.text:00401150 arg_0           = dword ptr  4
.text:00401150 arg_4           = dword ptr  8
.text:00401150
.text:00401150                 mov     eax, [esp+arg_0]
.text:00401154                 push    esi
.text:00401155                 mov     esi, [esp+4+arg_4]
.text:00401159                 mov     cl, [eax]
.text:0040115B                 mov     dl, [eax+1]
.text:0040115E                 shl     cl, 2
.text:00401161                 shr     dl, 4
.text:00401164                 or      cl, dl
.text:00401166                 mov     [esi], cl
.text:00401168                 mov     cl, [eax+2]
.text:0040116B                 mov     dl, [eax+1]
.text:0040116E                 shr     cl, 2
.text:00401171                 shl     dl, 4
.text:00401174                 or      cl, dl
.text:00401176                 mov     [esi+1], cl
.text:00401179                 mov     cl, [eax+2]
.text:0040117C                 mov     dl, [eax+3]
.text:0040117F                 shl     cl, 6
.text:00401182                 or      cl, dl
.text:00401184                 mov     [esi+2], cl
.text:00401187                 pop     esi
.text:00401188                 retn
.text:00401188 sub_401150      endp

sub_401150 takes two pointers and a rewrite to C would look something like this:

void sub_401150(char * first, char * second) {
    second[0] = (first[0] << 2) | (first[1] >> 4);
    second[1] = (first[1] << 4) | (first[2] >> 2);
    second[2] = (first[2] << 6) | (first[3]);
}

Here bits from four bytes from the first address are merged together and into three bytes in the second address.

That reminds me an awful lot about base64 decoding!

TO BE CONTINUED!!!