#===============================================================================
# Copyright 2015 Intel Corporation.
#
# This software and the related documents are Intel copyrighted  materials,  and
# your use of  them is  governed by the  express license  under which  they were
# provided to you (License).  Unless the License provides otherwise, you may not
# use, modify, copy, publish, distribute,  disclose or transmit this software or
# the related documents without Intel's prior written permission.
#
# This software and the related documents  are provided as  is,  with no express
# or implied  warranties,  other  than those  that are  expressly stated  in the
# License.
#===============================================================================

#  Content:
#      Makefile for Intel(R) oneMKL MPI wrapper library.
#      This library allows using custom MPI in Cluster Intel(R) oneMKL.
#*******************************************************************************

help:
	@echo
	@echo "Usage: make {libintel64|sointel64} [<options>]"
	@echo
	@echo "  libintel64"
	@echo "      A makefile target to build a custom BLACS static library to"
	@echo "      Intel(R) oneMKL for Intel(R) 64 architecture."
	@echo
	@echo "  sointel64"
	@echo "      A makefile target to build a custom BLACS dynamic library to"
	@echo "      Intel(R) oneMKL for Intel(R) 64 architecture."
	@echo
	@echo "  <options>"
	@echo
	@echo "    MPICC=<command>"
	@echo "        The specified wrapper compiler command will be used, for example,"
	@echo "        'mpicc', 'mpicc -cc=icx', 'mpiicx', etc."
	@echo "        Default: mpicc"
	@echo
	@echo "    interface={lp64|ilp64}"
	@echo "        The lp64 interface uses the 32-bit integer type, while the"
	@echo "        ilp64 interface uses the 64-bit integer type."
	@echo "        Default: lp64"
	@echo
	@echo "    MKLROOT=<path>"
	@echo "        Intel(R) oneMKL installation directory."
	@echo "        Default: ../../../.."
	@echo
	@echo "    INSTALL_DIR=<path>"
	@echo "        The built library will be installed in <path>."
	@echo "        Default: \$$(MKLROOT)/lib"
	@echo
	@echo "    INSTALL_LIBNAME=<name>"
	@echo "        The name of the built library without the '.a' or '.so' extension."
	@echo "        The default value depends on 'interface' in use, for example,"
	@echo "        'libmkl_blacs_custom_lp64'."
	@echo
	@echo "    CFLAGS=<flags>"
	@echo "        Additional compilation flags."
	@echo "        This variable is not predefined."
	@echo
	@echo "    CPPFLAGS=<flags>"
	@echo "        Additional preprocessor flags."
	@echo "        This variable is not predefined."
	@echo
	@echo "    TARGET_ARCH=<flags>"
	@echo "        Additional architecture-specific flags."
	@echo "        This variable is not predefined."
	@echo
	@echo "    AR=<program>"
	@echo "        The specified archive-maintaining program will be used."
	@echo "        Default: $(AR)"
	@echo
	@echo "    ARFLAGS=<flags>"
	@echo "        The specified modifier flags in addition to the 'u' modifier"
	@echo "        will be given to \$$(AR)."
	@echo "        Default: $(ARFLAGS)"
	@echo
	@echo "    LD=<linker>"
	@echo "        The specified linker will be used."
	@echo "        Default: ld"
	@echo
	@echo "Due to known issue in OpenMPI (https://github.com/open-mpi/ompi/issues/1763)"
	@echo "custom build for OpenMPI might not work properly."
	@echo "Use pre-built libmkl_blacs_openmpi_{lp64|ilp64}.{a|so} instead."
	@echo
	@echo
	@echo "Usage example:"
	@echo
	@echo "  make libintel64"
	@echo "      This command builds a custom BLACS static library to Intel(R) oneMKL"
	@echo "      for Intel(R) 64 architecture using an 'mpicc' wrapper compiler. The"
	@echo "      built static library 'libmkl_blacs_custom_lp64.a' is installed in"
	@echo "      ../../../../lib and uses the 32-bit integer type."
	@echo

#-------------------------------------------------------------------------------
## Please use the command line parameters to override the values below
MKLROOT ?= ../../../..
MPICC  ?= mpicc
LD ?= ld

ifeq ($(interface),ilp64)
    ILP_OPTS = -DMKL_ILP64
    ILP_EXT  = _ilp64
else
    ILP_OPTS =
    ILP_EXT  = _lp64
endif

ifndef INSTALL_DIR
    INSTALL_DIR = $(MKLROOT)/lib
    obj_path = ./obj_$(_IA)$(ILP_EXT)
else
    obj_path = $(INSTALL_DIR)/obj_$(_IA)$(ILP_EXT)
endif

override mpi = custom

INSTALL_LIBNAME ?= libmkl_blacs_$(mpi)$(ILP_EXT)
WRAPLIB_BLACS = $(INSTALL_LIBNAME)

.PHONY: libintel64

.PHONY: sointel64

libintel64:
	$(MAKE) clean mkobjdir wraplib ILP_EXT=$(ILP_EXT) _IA=intel64

sointel64:
	$(MAKE) clean mkobjdir wrapso ILP_EXT=$(ILP_EXT) _IA=intel64

#-------------------------------------------------------------------------------
# Nested makefile
#-------------------------------------------------------------------------------

ifdef _IA
    WRAP    = mklmpi-impl
    objs    = $(addsuffix .o ,$(WRAP))
    objects = $(addprefix $(obj_path)/,$(objs))

    ifeq (,$(findstring x$(interface),x xlp64 xilp64))
        $(warning "Using interface=lp64 instead of unknown $(interface)")
        ILP_OPTS =
        ILP_EXT  = _lp64
    endif

.SUFFIXES:

.SUFFIXES: .c .o

vpath %.c .

$(obj_path)/%.o: %.c
	$(MPICC) -c -Wall -fPIC $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) $(ILP_OPTS) -I$(MKLROOT)/include $< -o $@

wraplib: $(objects)
	cp $(MKLROOT)/lib/libmkl_blacs_intelmpi$(ILP_EXT).a $(obj_path)/libmkl_blacs_$(mpi)$(ILP_EXT).a
	cd $(obj_path) && $(AR) $(ARFLAGS)u libmkl_blacs_$(mpi)$(ILP_EXT).a $(objs)
	cp $(obj_path)/libmkl_blacs_$(mpi)$(ILP_EXT).a $(INSTALL_DIR)/$(WRAPLIB_BLACS).a

wrapso: $(objects)
	cp $(MKLROOT)/lib/libmkl_blacs_intelmpi$(ILP_EXT).a $(obj_path)/libmkl_blacs_$(mpi)$(ILP_EXT).a
	cd $(obj_path) && $(AR) $(ARFLAGS)u libmkl_blacs_$(mpi)$(ILP_EXT).a $(objs)
	$(LD) -shared  --whole-archive $(obj_path)/libmkl_blacs_$(mpi)$(ILP_EXT).a --no-whole-archive -o $(INSTALL_DIR)/$(WRAPLIB_BLACS).so

mkobjdir:
	-mkdir -p $(obj_path)
	-mkdir -p $(INSTALL_DIR)

clean:
	-rm -rf $(obj_path)
	-rm -f $(INSTALL_DIR)/$(WRAPLIB_BLACS)

endif
