How do I get CMake to compile a pure assembly static and shared library?

This is the library.
library.asm:

section .text

global return_number
return_number:
  mov eax, 10
  ret

I can easily make a shared and static library using the command line:

nasm -f elf64 -o library.o library.asm
ld -shared -o libreturn_number.so library.o
ar rcs libreturn_number.a library.o

However, using CMake presents problems.
CMakeLists.txt:

enable_language(ASM_NASM)

project(mylibrary)

set(CMAKE_ASM_NASM_CREATE_SHARED_LIBRARY "<CMAKE_ASM_NASM_COMPILER> <CMAKE_SHARED_LIBRARY_CREATE_ASM_NASM_FLAGS> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")

add_library(mylibrary_static STATIC library.asm)
add_library(mylibrary_shared SHARED library.asm)

The static library works perfectly fine, but the shared library throws a bunch of odd errors.

CMakeFiles/mylibrary_shared.dir/library.asm.o:1: error: label or instruction expected at start of line
CMakeFiles/mylibrary_shared.dir/library.asm.o:10: error: label or instruction expected at start of line
CMakeFiles/mylibrary_shared.dir/library.asm.o:11: error: label or instruction expected at start of line
CMakeFiles/mylibrary_shared.dir/library.asm.o:12: error: label or instruction expected at start of line
CMakeFiles/mylibrary_shared.dir/library.asm.o:31: warning: label alone on a line without a colon ...

for almost every single line in the '.o' file.

Customizing toolchain settings is best done in a toolchain file. You can write one for NASM and attach it to your project by default like so:

cmake/nasm-toolchain.cmake:

# Allow the user to override the object format at the command line
if (NOT DEFINED CMAKE_ASM_NASM_OBJECT_FORMAT)
  set(CMAKE_ASM_NASM_OBJECT_FORMAT elf64)
endif ()

# Determine the ld-compatible emulation mode flags, given the
# nasm output format
if (CMAKE_ASM_NASM_OBJECT_FORMAT MATCHES "elf64")
  set(extra_flags "-m elf_x86_64")
elseif (CMAKE_ASM_NASM_OBJECT_FORMAT MATCHES "elfx32")
  set(extra_flags "-m elf32_x86_64")
elseif (CMAKE_ASM_NASM_OBJECT_FORMAT MATCHES "elf(32)?")
  set(extra_flags "-m elf_i386")
else ()
  set(extra_flags "")
endif ()

set(CMAKE_ASM_NASM_CREATE_SHARED_LIBRARY
    "<CMAKE_LINKER> -shared ${extra_flags} <CMAKE_ASM_NASM_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")

Note the use of <CMAKE_LINKER> here to allow, e.g. lld or mold to be used.

CMakeLists.txt:

cmake_minimum_required(VERSION 3.25)

# Set the toolchain file by default because our use of NASM
# requires special configuration.
set(CMAKE_TOOLCHAIN_FILE "${CMAKE_CURRENT_LIST_DIR}/cmake/nasm-toolchain.cmake"
    CACHE STRING "")

project(example LANGUAGES ASM_NASM)

##
# Project code

add_library(mylib library.asm)

Note that for many reasons it is better to avoid creating separate targets for shared and static libraries. Use BUILD_SHARED_LIBS instead.

library.asm

Same as above, but reproduced here for completeness:

section .text

global return_number
return_number:
  mov eax, 10
  ret

Build output:

Shared library (64 bit):

$ cmake -G Ninja -S . -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=YES
-- The ASM_NASM compiler identification is NASM
-- Found assembler: /usr/bin/nasm
-- Configuring done
-- Generating done
-- Build files have been written to: /home/alex/test/build
$ cmake --build build/ --verbose
[1/2] /usr/bin/nasm -Dmylib_EXPORTS   -MD CMakeFiles/mylib.dir/library.asm.o.d -MT CMakeFiles/mylib.dir/library.asm.o -f elf64 -o CMakeFiles/mylib.dir/library.asm.o /home/alex/test/library.asm
[2/2] : && /usr/bin/ld -shared -m elf_x86_64  CMakeFiles/mylib.dir/library.asm.o -o libmylib.so   && :

Shared library (32 bit):

$ cmake -G Ninja -S . -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=YES -DCMAKE_ASM_NASM_OBJECT_FORMAT=elf32
-- The ASM_NASM compiler identification is NASM
-- Found assembler: /usr/bin/nasm
-- Configuring done
-- Generating done
-- Build files have been written to: /home/alex/test/build
$ cmake --build build/ --verbose
[1/2] /usr/bin/nasm -Dmylib_EXPORTS   -MD CMakeFiles/mylib.dir/library.asm.o.d -MT CMakeFiles/mylib.dir/library.asm.o -f elf32 -o CMakeFiles/mylib.dir/library.asm.o /home/alex/test/library.asm
[2/2] : && /usr/bin/ld -shared -m elf_i386  CMakeFiles/mylib.dir/library.asm.o -o libmylib.so   && :

Static library:

$ cmake -G Ninja -S . -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=NO
-- The ASM_NASM compiler identification is NASM
-- Found assembler: /usr/bin/nasm
-- Configuring done
-- Generating done
-- Build files have been written to: /home/alex/test/build
$ cmake --build build/ --verbose
[1/2] /usr/bin/nasm    -MD CMakeFiles/mylib.dir/library.asm.o.d -MT CMakeFiles/mylib.dir/library.asm.o -f elf64 -o CMakeFiles/mylib.dir/library.asm.o /home/alex/test/library.asm
[2/2] : && /usr/bin/ar cr libmylib.a  CMakeFiles/mylib.dir/library.asm.o  && /usr/bin/ranlib libmylib.a && :

The problem is that you are trying to use the nasm compiler to link the object binaries. You should use the ld linker or even gcc for that. So nasm is reading the object files and saying wtf is this!

The solution below is using the same asm file you presented above.

For 64-bits the CMakeLists.txt is

cmake_minimum_required(VERSION 3.12)
set(CMAKE_ASM_NASM_CREATE_SHARED_LIBRARY
  "ld -lc -shared  <CMAKE_ASM_NASM_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
set(CMAKE_ASM_NASM_OBJECT_FORMAT elf64)

project(mylibrary)

enable_language(ASM_NASM)

add_library(mylibrary_shared SHARED library.asm)

$ cmake ..
-- The ASM_NASM compiler identification is NASM
-- Found assembler: /usr/bin/nasm
-- The C compiler identification is GNU 9.4.0
-- The CXX compiler identification is GNU 9.4.0
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Check for working C compiler: /usr/bin/cc - skipped
-- Detecting C compile features
-- Detecting C compile features - done
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Check for working CXX compiler: /usr/bin/c++ - skipped
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Configuring done
-- Generating done
-- Build files have been written to: ../build
$ make
-- Configuring done
-- Generating done
-- Build files have been written to: /home/hbucher/git/nasm/build
[ 50%] Building ASM_NASM object CMakeFiles/mylibrary_shared.dir/library.asm.o
[100%] Linking ASM_NASM shared library libmylibrary_shared.so
$ ldd libmylibrary_shared.so 
        linux-vdso.so.1 (0x00007ffffb59e000)
        libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f8bf8822000)
        /lib64/ld-linux-x86-64.so.2 (0x00007f8bf8a4a000)

This for 32 bits:

cmake_minimum_required(VERSION 3.12)
set(CMAKE_ASM_NASM_CREATE_SHARED_LIBRARY
  "ld -lc -shared -m elf_i386 <CMAKE_ASM_NASM_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")

project(mylibrary)

enable_language(ASM_NASM)

add_library(mylibrary_shared SHARED library.asm)

Produces

$ cmake ..
-- The ASM_NASM compiler identification is NASM
-- Found assembler: /usr/bin/nasm
-- The C compiler identification is GNU 9.4.0
-- The CXX compiler identification is GNU 9.4.0
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Check for working C compiler: /usr/bin/cc - skipped
-- Detecting C compile features
-- Detecting C compile features - done
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Check for working CXX compiler: /usr/bin/c++ - skipped
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Configuring done
-- Generating done
$ make 
[ 50%] Building ASM_NASM object CMakeFiles/mylibrary_shared.dir/library.asm.o
[100%] Linking ASM_NASM shared library libmylibrary_shared.so
[100%] Built target mylibrary_shared
$ ldd libmylibrary_shared.so 
        linux-gate.so.1 (0xf7f39000)
        libc.so.6 => /lib/i386-linux-gnu/libc.so.6 (0xf7d10000)
        /lib/ld-linux.so.2 (0xf7f3b000)