General Commentary

I don’t usually work with malware these days. FLARE-ON is probably the only time I do reverse engineering all year. This edition had mostly analysis, both dynamic and static, plus some math and problem solving. I really enjoy these challenges. I start clueless but once I begin breaking things down, it keeps me engaged through weekends and nights.

I have changed the decompilation a bit to make it easier to understand at times.

Challenge 9 - 10000

$ file 10000.exe
10000.exe: PE32+ executable (console) x86-64 (stripped to external PDB), for MS Windows, 10 sections
$ ls -lah 10000.exe
Permissions Size User      Date Modified Name
.rw-r--r--  1.2G sudhackar 24 Sep 21:08  10000.exe

A 1.2GB PE file, as the name suggests we have 10000 - yes thats 10k resources

$ peres -s 10000.exe
Total Structs:                   40003
Total Resource Directory:        10002
Total Directory Entry:           20001
Total Data String:               0
Total Data Entry:                10000
$ readpe -h optional 10000.exe
Optional/Image header
    Magic number:                    0x20b (PE32+)
    Linker major version:            2
    Linker minor version:            44
    Size of .text section:           0xcaa00

The code itself is pretty small - the obfuscation is nil and its pretty chill to figure out. I think the core theme of this challenge was not RE but problem solving.

  v0 = cout_str(qword_1400D8800, "checking license file...");
  ...
  open((__int64)license_fd, (__int64)"license.bin", v1);
  dup2((__int64)v16, license_fd);
  byte_size_license = get_size((__int64)v16);
  if ( byte_size_license == 340000 )
  {
    license_buf = malloc_0(byte_size_license);
    read(license_fd, license_buf, byte_size_license, v4);
    get_sha_256(license_buf, license_buf + 340000, sha256_license, license_fd);
    curr_ptr = (check *)license_buf;
    ...
    for ( i = 0; i <= 9999; ++i )
    {
      dll_number = curr_ptr->dll_number;
      if ( dll_number > 9999u )
        goto fail;
      ...
      dll_recursive = load_dll_recursive(dll_number);
      p_export_table = &dll_recursive->export_table;
      make_str(check_str, "_Z5checkPh");
      check_fn_dll = (int (__fastcall **)(char *))find_export(p_export_table, check_str);
      LODWORD(p_export_table) = ((__int64 (__fastcall *)(unsigned __int16 *))*check_fn_dll)(curr_ptr->buf) ^ 1;
      destroy_str((__int64)check_str);
      if ( (_BYTE)p_export_table )
        goto fail;
      ++curr_ptr;
      end_iter(i);
    }
    if ( memcmp(licence_check_buf, correct_license_buf, 40000u) )
    {
fail:
      v7 = cout_str(qword_1400D8800, "invalid license file");
      print_stdout(v7);
      ...
    }
    v8 = cout_str(qword_1400D8800, "license valid!");
    print_stdout(v8);
    ....
    aes_decrypt(encrypted_flag, 0x50u, sha256_license, 0x20u, iv_maybe, &flag__text[4], v11, flag__text);
    v9 = sub_1400C76E0(qword_1400D8800, &flag__text[4]);
    print_stdout(v9);

The main program is a license checker - its simple to decompile and follow. It reads a 34k bytes binary file - reads it in multiples of 34 - 2 bytes for the DLL number and 32 bytes is passed to the check function exported in the DLL. All 10k of these checks need to pass for license to be valid.

It calls load_dll_recursive with dll_number which loads the DLL in the resources. All the dependencies are recursively parsed and loaded at the same time - each DLL in the resources is also dependent on a few functions in other DLLs.

The DLLs are cached in a struct - so at the start it check if a DLL is already loaded

  end = iter_end(vec_instance_dll);
  start = iter_begin(vec_instance_dll);
  found_index = find_in_container(start, end, dll_number);
  v13 = iter_end(vec_instance_dll);
  if ( !iter_eq((__int64)&found_index, (__int64)&v13) )
    return *(dll_obj **)vec_get_obj_at((__int64)&found_index);

It then loads the DLL and calls itself recursively - the DLL is compressed with aplib

  hResInfo = FindResourceA(0, (LPCSTR)dll_number, (LPCSTR)0xA);
  hResData = LoadResource(0, hResInfo);
  v46 = (unsigned __int8 *)LockResource(hResData);
  resource_size = SizeofResource(0, hResInfo);
  decompressed_sz = get_decompressed_sz(v46, resource_size, 0);
  sub_1400C0C30(v10, decompressed_sz);
  v43 = (PIMAGE_DOS_HEADER)alloc_array((__int64)v10, 0);
  sub_1400035E8(v46, (unsigned __int64)v43, resource_size, decompressed_sz, 0);
  v42 = (PIMAGE_NT_HEADERS)((char *)v43 + v43->e_lfanew);
  pe_buf = (char *)VirtualAlloc(0, v42->OptionalHeader.SizeOfImage, 0x3000u, 0x40u);
  dll_obj->pe_buf = pe_buf;
  v55 = (PIMAGE_SECTION_HEADER)((char *)&v42->OptionalHeader + v42->FileHeader.SizeOfOptionalHeader);
  for ( i = 0; i < v42->FileHeader.NumberOfSections; ++i )
  {
    memcpy(&pe_buf[v55->VirtualAddress], (char *)v43 + v55->PointerToRawData, v55->SizeOfRawData);
    ++v55;
  }
  for ( j = (PIMAGE_IMPORT_DESCRIPTOR)&pe_buf[v42->OptionalHeader.DataDirectory[1].VirtualAddress]; j->Name; ++j )
  {
    import_dll_name = &pe_buf[j->Name];
    entry = (FARPROC *)&pe_buf[j->FirstThunk];
    if ( *import_dll_name - (unsigned int)'0' > 9// load local DLLs
      || import_dll_name[1] - (unsigned int)'0' > 9
      || import_dll_name[2] - (unsigned int)'0' > 9
      || import_dll_name[3] - (unsigned int)'0' > 9 )
    {
      hModule = LoadLibraryA(import_dll_name);
      while ( *entry )
      {
        proc_name = &pe_buf[(_QWORD)*entry];
        ProcAddress = GetProcAddress(hModule, proc_name + 2);
        *entry++ = ProcAddress;
      }
    }
    else
    {
      dependency_dll_number = atoi(import_dll_name);// load PE resource DLLs
      dll_recursive = load_dll_recursive(dependency_dll_number);
      while ( *entry )
      {
        proc_name_1 = &pe_buf[(_QWORD)*entry];
        p_export_table = &dll_recursive->export_table;
        v19 = &v15;
        make_str(name, proc_name_1 + 2);
        fn_ptr = *(INT_PTR (__stdcall **)())find_export(p_export_table, name);
        destroy_str((__int64)name);
        *entry++ = fn_ptr;
      }
    }
  }

After this the entrypoint to the dll is called and the instance is cached

  entrypoint = (__int64 (__fastcall *)(_DWORD *, __int64, _QWORD))&pe_buf[v42->OptionalHeader.AddressOfEntryPoint];
  v32 = entrypoint(licence_check_buf, 1, 0);
  vec_push_back(vec_instance_dll, (__int64)&dll_obj);

It is to note that the licence_check_buf is passed as first arg to the entrypoint. At the end of every iteration of the 10k steps

__int64 __fastcall end_iter(int iteration_count)
{

  v5 = vec_instance_dll;
  v3 = iter_begin(vec_instance_dll);
  v2 = iter_end((__int64)vec_instance_dll);
  while ( !iter_eq((__int64)&v3, (__int64)&v2) )
  {
    dll = *(dll_obj **)vec_get_obj_at((__int64)&v3);
    licence_check_buf[dll->dll_number] += iteration_count;
    VirtualFree((LPVOID)dll->pe_buf, 0, 0x8000u);
    iter_next(&v3);
  }
  return sub_1400AF7B0(vec_instance_dll);
}

All the DLLs loaded in the current iteration are unmapped and