This crackme can be downloaded from here.

I run the program which asks for a serial number. Writing one and pressing 'Check' does not seem to do anything so I load the program in IDA Pro, where I locate the GetDlgItemTextA function in the import list and finds all references to it. The only one is in sub_401400 shown below:

.text:00401400 sub_401400      proc near               ; DATA XREF: .rdata:004023C4o
.text:00401400
.text:00401400 var_2C          = dword ptr -2Ch
.text:00401400 var_20          = byte ptr -20h
.text:00401400
.text:00401400                 sub     esp, 2Ch
.text:00401403                 lea     eax, [esp+2Ch+var_2C]
.text:00401407                 push    ebp
.text:00401408                 push    8
.text:0040140A                 push    eax
.text:0040140B                 mov     ebp, ecx
.text:0040140D                 push    3E8h
.text:00401412                 call    ?GetDlgItemTextA@CWnd@@QBEHHPADH@Z ; CWnd::GetDlgItemTextA(int,char *,int)
.text:00401417                 test    eax, eax
.text:00401419                 jz      loc_4014A5
.text:0040141F                 push    ebx
.text:00401420                 push    esi
.text:00401421                 lea     ecx, [esp+38h+var_2C]
.text:00401425                 push    edi
.text:00401426                 push    ecx
.text:00401427                 call    sub_401320
.text:0040142C                 lea     edx, [esp+40h+var_2C]
.text:00401430                 mov     esi, eax
.text:00401432                 push    edx
.text:00401433                 call    sub_401360
.text:00401438                 mov     edi, eax
.text:0040143A                 lea     eax, [esp+44h+var_2C]
.text:0040143E                 push    eax
.text:0040143F                 call    sub_4013C0
.text:00401444                 push    edi
.text:00401445                 push    eax
.text:00401446                 push    esi
.text:00401447                 lea     ecx, [esp+54h+var_20]
.text:0040144B                 push    offset a08x08x08x ; "%08X-%08X-%08X"
.text:00401450                 push    ecx             ; char *
.text:00401451                 call    ds:sprintf
.text:00401457                 add     esp, 20h
.text:0040145A                 mov     esi, offset a0ceac848A1f552 ; "0CEAC848-A1F55297-18B3DD62"
.text:0040145F                 lea     eax, [esp+3Ch+var_20]
.text:00401463
.text:00401463 loc_401463:                             ; CODE XREF: sub_401400+85j
.text:00401463                 mov     dl, [eax]
.text:00401465                 mov     bl, [esi]
.text:00401467                 mov     cl, dl
.text:00401469                 cmp     dl, bl
.text:0040146B                 jnz     short loc_40148B
.text:0040146D                 test    cl, cl
.text:0040146F                 jz      short loc_401487
.text:00401471                 mov     dl, [eax+1]
.text:00401474                 mov     bl, [esi+1]
.text:00401477                 mov     cl, dl
.text:00401479                 cmp     dl, bl
.text:0040147B                 jnz     short loc_40148B
.text:0040147D                 add     eax, 2
.text:00401480                 add     esi, 2
.text:00401483                 test    cl, cl
.text:00401485                 jnz     short loc_401463
.text:00401487
.text:00401487 loc_401487:                             ; CODE XREF: sub_401400+6Fj
.text:00401487                 xor     eax, eax
.text:00401489                 jmp     short loc_401490
.text:0040148B ; ---------------------------------------------------------------------------
.text:0040148B
.text:0040148B loc_40148B:                             ; CODE XREF: sub_401400+6Bj
.text:0040148B                                         ; sub_401400+7Bj
.text:0040148B                 sbb     eax, eax
.text:0040148D                 sbb     eax, 0FFFFFFFFh
.text:00401490
.text:00401490 loc_401490:                             ; CODE XREF: sub_401400+89j
.text:00401490                 pop     edi
.text:00401491                 pop     esi
.text:00401492                 test    eax, eax
.text:00401494                 pop     ebx
.text:00401495                 jnz     short loc_4014A5
.text:00401497                 push    eax
.text:00401498                 push    eax
.text:00401499                 push    offset aSerialIsCorrec ; "Serial is Correct!!!"
.text:0040149E                 mov     ecx, ebp
.text:004014A0                 call    ?MessageBoxA@CWnd@@QAEHPBD0I@Z ; CWnd::MessageBoxA(char const *,char const *,uint)
.text:004014A5
.text:004014A5 loc_4014A5:                             ; CODE XREF: sub_401400+19j
.text:004014A5                                         ; sub_401400+95j
.text:004014A5                 pop     ebp
.text:004014A6                 add     esp, 2Ch
.text:004014A9                 retn
.text:004014A9 sub_401400      endp

The function starts by retrieving the content of the textfield to a local variable (address 00401412). Then three functions are called: sub_401320, sub_401360 and sub_4013C0. They all get the serial as parameter. Actually they get a lot of things passed to them, but they only use the serial. The stack is cleaned up at address 00401457.

The return value from sub_401320 is put in the ESI register, the return value from sub_401360 in EDI and the result from sub_4013c0 stays in EAX.

From 00401444 till 00401446 EDI, EAX and ESI are pushed onto the stack and sprintf is called with the format string "%08X-%08X-%08X".

The result from this is compared against "0CEAC848-A1F55297-18B3DD62" two characters at a time.

So to brute force this we need to rip the three functions and make them return the following:

Function: Return value:
sub_401320 0x0CEAC848
sub_401360 0x18B3DD62
sub_4013C0 0xA1F55297

sub_401320

sub_401320 ripped and modified to working nasm code:

;Filename: sub_401320.asm
SECTION .text
BITS 32
global sub_401320
 
sub_401320:
                push    ebp
                mov     ebp, esp
                push    esi
                mov     esi, [ebp+8]
                xor     eax, eax
                mov     cl, [esi]
                test    cl, cl
                jz      short loc_40135E
                push    edi
 
loc_40132E:
                movsx   ecx, cl
                mov     edi, ecx
                xor     edi, 0x0BAD0BEEF
                add     edi, eax
                mov     eax, ecx
                and     eax, 0x0FEFEFEFE
                or      edi, eax
                mov     eax, ecx
                cdq
                mov     ecx, 0x0D
                idiv    ecx
                not     edi
                mov     cl, [esi+1]
                imul    edi, edx
                inc     esi
                mov     eax, edi
                test    cl, cl
                jnz     short loc_40132E
                pop     edi
 
loc_40135E:
                pop     esi
                mov     esp, ebp
                pop     ebp
                ret

sub_401360

sub_401360 ripped and modified to working nasm code:

;Filename: sub_401360.asm
SECTION .text
BITS 32
global sub_401360
 
sub_401360:
                push    ebp
                mov     ebp, esp
                push    esi
                mov     esi, [ebp+8]
                xor     eax, eax
                mov     cl, [esi]
                test    cl, cl
                jz      short loc_4013B8
                push    ebx
                push    edi
 
loc_40136F:
                movsx   ecx, cl
                mov     edi, ecx
                mov     ebx, 0x11
                xor     edi, 0x0B00BFACE
                add     edi, eax
                mov     eax, ecx
                or      eax, 0x0FEFEFEFE
                and     edi, eax
                lea     eax, [ecx+ecx*2]
                shl     eax, 5
                add     eax, ecx
                lea     eax, [eax+eax*8]
                shl     eax, 2
                cdq
                idiv    ebx
                mov     eax, ecx
                mov     ecx, 5
                add     edi, edx
                cdq
                idiv    ecx
                add     edx, ecx
                mov     cl, [esi+1]
                imul    edi, edx
                inc     esi
                mov     eax, edi
                test    cl, cl
                jnz     short loc_40136F
                pop     edi
                pop     ebx
 
loc_4013B8:
                pop     esi
                mov     esp, ebp
                pop     ebp
                ret

sub_4013C0

sub_4013C0 ripped and modified to working nasm code:

;Filename: sub_4013C0.asm
SECTION .text
BITS 32
global sub_4013C0
 
sub_4013C0:
                push    ebp
                mov     ebp, esp
                mov     edx, [ebp+8]
                xor     eax, eax
                mov     cl, [edx]
                test    cl, cl
                jz      short locret_4013F1
                push    esi
 
loc_4013CD:
                movsx   ecx, cl
                mov     esi, ecx
                xor     esi, 0x0BAD0BEEF
                add     esi, eax
                mov     eax, ecx
                or      eax, 0x0DEADC0DE
                xor     esi, eax
                imul    esi, ecx
                mov     cl, [edx+1]
                inc     edx
                test    cl, cl
                mov     eax, esi
                jnz     short loc_4013CD
                pop     esi
 
locret_4013F1:
                mov     esp, ebp
                pop     ebp
                ret

The brute forcer

Now that I have the code for the hashing functions I code up a routing to use them for brute forcing the serial. For this I use my PasswordIterator library:

/**
 * Filename: nts-crackme4-brute.c
 */
#include <stdio.h>
#include "pi.h"
 
int sub_401320(char *);
int sub_401360(char *);
int sub_4013C0(char *);
 
int main (int argc, char ** argv) {
    char current[32];
    PI pi;
    pi_initialize(&pi, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789");
    while (pi_next(&pi, current, sizeof(current))) {
        if (sub_401320(current) == 0x0CEAC848 &&
            sub_401360(current) == 0x18B3DD62 &&
            sub_4013C0(current) == 0xA1F55297) {
            printf("Hoooray, \"%s\" is the correct serial.\n", current);
            return 0;
        }
    }
    return -1;
}

And a makefile for building:

#Filename: Makefile
C_FILES=$(wildcard *.c)
ASM_FILES=$(wildcard *.asm)
OBJ_FILES=$(C_FILES:.c=.o) $(ASM_FILES:.asm=.o)
EXECUTABLE=nts-crackme4-brute

.PHONY: all
all: $(EXECUTABLE)

.PHONY: run
run: all
  ./$(EXECUTABLE)

$(EXECUTABLE): $(OBJ_FILES)
  gcc -o $@ $^

.PHONY: clean
clean:
  rm -f $(OBJ_FILES) $(EXECUTABLE)

%.o: %.asm
  nasm -f elf -o $@ $<

%.o: %.c
  gcc -c -o $@ $<

And we execute:

robert-laptop:~/nts-crackme4 $ make run
gcc -c -o nts-crackme4-brute.o nts-crackme4-brute.c
gcc -c -o pi.o pi.c
nasm -f elf -o sub_401320.o sub_401320.asm
nasm -f elf -o sub_401360.o sub_401360.asm
nasm -f elf -o sub_4013C0.o sub_4013C0.asm
gcc -o nts-crackme4-brute nts-crackme4-brute.o pi.o sub_401320.o sub_401360.o sub_4013C0.o
./nts-crackme4-brute
Hoooray, "ycc" is the correct serial.
robert-laptop:~/nts-crackme4 $