From 92b422ed001c1f552c458b693e01c4d4bb41312a Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Fri, 1 Apr 2022 22:04:20 +0200 Subject: [PATCH 01/21] Add documentation to openssl Android toolchain Add a link of possible interest for future updates --- misc/Android/prepare-toolchain-clang.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misc/Android/prepare-toolchain-clang.sh b/misc/Android/prepare-toolchain-clang.sh index fd2a60a6e..358ed2970 100755 --- a/misc/Android/prepare-toolchain-clang.sh +++ b/misc/Android/prepare-toolchain-clang.sh @@ -401,6 +401,8 @@ build_bzlib() )} ## More information available at http://doc.qt.io/qt-5/opensslsupport.html +## The following article might be interesting for future updates +## https://proandroiddev.com/tutorial-compile-openssl-to-1-1-1-for-android-application-87137968fee task_register build_openssl build_openssl() {( set -e From a574263c2448ba156111c443e7dcca86ef6e9e7f Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Fri, 1 Apr 2022 22:05:26 +0200 Subject: [PATCH 02/21] Fix default installation paths via GNUInstallDirs --- CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c5cfff044..e6a4cd135 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,7 @@ cmake_minimum_required (VERSION 3.18.0) project(retroshare) include(CMakeDependentOption) +include(GNUInstallDirs) set(FETCHCONTENT_QUIET OFF) include(FetchContent) @@ -122,19 +123,19 @@ option( set( RS_DATA_DIR - "${CMAKE_INSTALL_PREFIX}/share/${PROJECT_NAME}" + "${CMAKE_INSTALL_FULL_DATADIR}/${PROJECT_NAME}" CACHE PATH "Path where to install RetroShare system wide data" ) set( RS_INCLUDE_INSTALL_DIR - "${CMAKE_INSTALL_PREFIX}/include/${PROJECT_NAME}" + "${CMAKE_INSTALL_FULL_INCLUDEDIR}/${PROJECT_NAME}" CACHE PATH "Path where to install libretroshare headers" ) set( RS_LIB_INSTALL_DIR - "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}" + "${CMAKE_INSTALL_FULL_LIBDIR}" CACHE PATH "Path where to install libretroshare compiled library" ) From 202642c5bcf5d3fe1967089424146c2d5f5d0ece Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Fri, 1 Apr 2022 22:06:52 +0200 Subject: [PATCH 03/21] Gradle initial support for separated debug and release builds Fail noticeably if copying of library or assets fails --- build.gradle | 130 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 88 insertions(+), 42 deletions(-) diff --git a/build.gradle b/build.gradle index 5b6aeb5ea..c720a8bd8 100644 --- a/build.gradle +++ b/build.gradle @@ -125,8 +125,9 @@ ext.getRsExtraCmakeOpts = } ext.getjniLibsDir = -{ -> - return "${buildDir}/native_libs/" +{ + pBuildType -> + return "${buildDir}/native_libs/${pBuildType.toLowerCase()}" } ext.getAssetsDir = @@ -135,15 +136,31 @@ ext.getAssetsDir = } ext.getArtifactBaseName = -{ -> +{ -> return project.name + "-MinApiLevel" + getAndroidMinApiLevel() } +ext.copyChecked = +{ + sourceFile, + destDir -> + + assert file(sourceFile).exists() + assert file(destDir).exists() + copy + { + from sourceFile + into destDir + } + assert file("$destDir/${file(sourceFile).getName()}").exists() +} + ext.buildLibretroshareNativeLib = { pApiLevel, /* Android API level */ pAbi, /* Arch name as seen in AAR native libraries directories */ pNdkPath, /* Android NDK path */ + pBuildType, /* Debug, Release... */ pReuseToolchain = true /* If true reuse previously built toochain */ -> /* Convert pAbi into corresponding prepare toolchain ANDROID_NDK_ARCH */ @@ -168,7 +185,7 @@ ext.buildLibretroshareNativeLib = def toolchainsWorkdir = getNativeToolchainsDir() mkdir toolchainsWorkdir - def currToolchainPath = "$toolchainsWorkdir/$pApiLevel-$toolchainArch/" + def currToolchainPath = "$toolchainsWorkdir/$pApiLevel-$toolchainArch-${pBuildType.toLowerCase()}/" def toolChainScriptPath = "${projectDir}/misc/Android/prepare-toolchain-clang.sh" @@ -199,51 +216,30 @@ ext.buildLibretroshareNativeLib = } } - mkdir getjniLibsDir() + def currJniLibsDir = getjniLibsDir(pBuildType) + mkdir currJniLibsDir - def currAbiLibDir = "${getjniLibsDir()}/$pAbi/" + def currAbiLibDir = "$currJniLibsDir/$pAbi/" mkdir currAbiLibDir - copy - { - from "${currToolchainPath}/sysroot/usr/lib/libretroshare.so" - into currAbiLibDir - } + copyChecked( + "${currToolchainPath}/sysroot/usr/lib/libretroshare.so", + currAbiLibDir ) - copy - { - from "${currToolchainPath}/sysroot/usr/lib/${libcxxsharedTriple}/libc++_shared.so" - into currAbiLibDir - } + copyChecked( + "${currToolchainPath}/sysroot/usr/lib/${libcxxsharedTriple}/libc++_shared.so", + currAbiLibDir ) - copy - { - from "${currToolchainPath}/sysroot/usr/lib/${libcxxsharedTriple}/${pApiLevel}/liblog.so" - into currAbiLibDir - } + copyChecked( + "${currToolchainPath}/sysroot/usr/lib/${libcxxsharedTriple}/${pApiLevel}/liblog.so", + currAbiLibDir ) def bdbootAssetDir = "${getAssetsDir()}/values" mkdir bdbootAssetDir - copy - { - from "${currToolchainPath}/sysroot/usr/share/retroshare/bdboot.txt" - into bdbootAssetDir - } + copyChecked( + "${currToolchainPath}/sysroot/usr/share/retroshare/bdboot.txt", + bdbootAssetDir ) } -task prepareRetroshareNativeLibs -{ - doLast - { - def pNdkPath = android.getNdkDirectory().getAbsolutePath() - getJniNativeLibsArchs().each - { - abi -> - buildLibretroshareNativeLib(getAndroidMinApiLevel(), abi, pNdkPath) - } - } -} -preBuild.dependsOn prepareRetroshareNativeLibs - android { // see https://stackoverflow.com/questions/27301867/what-is-compilesdkversion @@ -263,6 +259,47 @@ android setProperty("archivesBaseName", getArtifactBaseName()) + buildTypes.all + { + buildTypeObj -> + + def buildTypeName = buildTypeObj.name.capitalize() + def preTaskName = "pre${buildTypeName}Build" + def buildNativeTaskName = "build${buildTypeName}RetroshareNativeLibs" + + tasks.register(buildNativeTaskName) + { + doLast + { + delete getjniLibsDir(buildTypeName) + + def pNdkPath = android.getNdkDirectory().getAbsolutePath() + getJniNativeLibsArchs().each + { + abi -> + buildLibretroshareNativeLib( + getAndroidMinApiLevel(), abi, pNdkPath, buildTypeName ) + } + } + } + + /* While `preBuild` seems always defined so one can simply use + * `preBuild.dependsOn prepareRetroshareNativeLibs` it is not the same with + * `preDebugBuild` or any `pre${buildType}Build` that gives this error + * if used directly + * "Could not get unknown property 'preDebugBuild' for root project 'libretroshare' of type org.gradle.api.Project" + * So make them depend on native build once they are added + */ + tasks.whenTaskAdded + { + task -> + if(task.name == preTaskName) + { + task.dependsOn buildNativeTaskName + } + } + } + sourceSets { main @@ -270,7 +307,16 @@ android java.srcDirs = [ 'src/rs_android/' ] manifest.srcFile 'src/rs_android/AndroidManifest.xml' assets.srcDirs = [ getAssetsDir() ] - jniLibs.srcDirs = [ getjniLibsDir() ] + } + + debug + { + jniLibs.srcDirs = [ getjniLibsDir(name) ] // name == "debug" + } + + release + { + jniLibs.srcDirs = [ getjniLibsDir(name) ] } } @@ -300,7 +346,7 @@ afterEvaluate groupId = 'org.retroshare.service' artifactId "${getArtifactBaseName()}-debug" version getLibretroshareSourceVersion() - artifact bundleReleaseAar + artifact bundleDebugAar } release(MavenPublication) { From 9bd204d3c274942e235aa8173537635eb68a151c Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Sat, 2 Apr 2022 22:09:48 +0200 Subject: [PATCH 04/21] Filter GitlabCI AAR jobs Those jobs need extra storage that shared runners doesn't provide, run only for project that export a variable that inform about extended storage runners avalibaility, avoid false pipeline failures on forks --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 76fb3e4b7..e3b44f67c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,6 @@ android-AAR-minapi16: + rules: + - if: $PROJECT_HAVE_STORAGE50_RUNNERS == "true" image: registry.gitlab.com/retroshare/retroshare:android_aar_base_16 script: - > @@ -7,8 +9,12 @@ android-AAR-minapi16: -PANDROID_MIN_API_LEVEL=16 -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + tags: + - storage50 android-AAR-minapi21: + rules: + - if: $PROJECT_HAVE_STORAGE50_RUNNERS == "true" image: registry.gitlab.com/retroshare/retroshare:android_aar_base_21 script: - > @@ -21,6 +27,8 @@ android-AAR-minapi21: - storage50 android-AAR-minapi24: + rules: + - if: $PROJECT_HAVE_STORAGE50_RUNNERS == "true" image: registry.gitlab.com/retroshare/retroshare:android_aar_base_24 script: - > From bcc4f087e293fd4d31e7d3cb3d9847c3b4516936 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Sun, 3 Apr 2022 19:08:12 +0200 Subject: [PATCH 05/21] Android pass build type Gradle > Toolchain > CMake --- build.gradle | 2 ++ misc/Android/prepare-toolchain-clang.sh | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/build.gradle b/build.gradle index c720a8bd8..f762fc45a 100644 --- a/build.gradle +++ b/build.gradle @@ -199,6 +199,7 @@ ext.buildLibretroshareNativeLib = environment "ANDROID_PLATFORM_VER", pApiLevel environment "ANDROID_NDK_ARCH", toolchainArch environment "RS_EXTRA_CMAKE_OPTS", getRsExtraCmakeOpts() + environment "TOOLCHAIN_BUILD_TYPE", pBuildType commandLine toolChainScriptPath } } @@ -212,6 +213,7 @@ ext.buildLibretroshareNativeLib = environment "ANDROID_PLATFORM_VER", pApiLevel environment "ANDROID_NDK_ARCH", toolchainArch environment "RS_EXTRA_CMAKE_OPTS", getRsExtraCmakeOpts() + environment "TOOLCHAIN_BUILD_TYPE", pBuildType commandLine toolChainScriptPath, 'build_libretroshare' } } diff --git a/misc/Android/prepare-toolchain-clang.sh b/misc/Android/prepare-toolchain-clang.sh index 358ed2970..7e5dbf8a5 100755 --- a/misc/Android/prepare-toolchain-clang.sh +++ b/misc/Android/prepare-toolchain-clang.sh @@ -106,6 +106,8 @@ define_default_value REPORT_DIR "$(pwd)/$(basename ${NATIVE_LIBS_TOOLCHAIN_PATH} define_default_value RS_SRC_DIR "$(realpath $(dirname $BASH_SOURCE)/../../)" define_default_value RS_EXTRA_CMAKE_OPTS "" +# Debug or Release we should give support at least at those two builds type supported by CMake +define_default_value TOOLCHAIN_BUILD_TYPE "" cArch="" eABI="" @@ -206,8 +208,13 @@ function andro_cmake() esac _hi="$HOST_IGNORE_PREFIX" + + cmakeBuildType="" + [ "$TOOLCHAIN_BUILD_TYPE" == "" ] || + cmakeBuildType="-DCMAKE_BUILD_TYPE=$TOOLCHAIN_BUILD_TYPE" cmake \ + $cmakeBuildType \ -DCMAKE_SYSTEM_PROCESSOR=$cmakeProc \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ -DCMAKE_PREFIX_PATH="${PREFIX}" \ From f0a88b3d4d3bb46a51416cfdf2dc9f9977cb77b2 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Tue, 5 Apr 2022 10:31:42 +0200 Subject: [PATCH 06/21] Initial support for Android CI on shared runners --- .gitlab-ci.yml | 14 +++++++++++++- misc/Android/Dockerfile | 21 ++++++++++++++------- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e3b44f67c..fec503aec 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ android-AAR-minapi16: rules: - if: $PROJECT_HAVE_STORAGE50_RUNNERS == "true" - image: registry.gitlab.com/retroshare/retroshare:android_aar_base_16 + image: registry.gitlab.com/retroshare/retroshare:android_aar_base_16_build script: - > ./gradlew build publish @@ -39,3 +39,15 @@ android-AAR-minapi24: -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" tags: - storage50 + +android-minapi16_armeabi-v7a_bundleDebugAar: + rules: + - if: $PROJECT_HAVE_STORAGE50_RUNNERS != "true" + image: registry.gitlab.com/retroshare/retroshare:android_aar_base_16_armeabi-v7a_bundleDebugAar + script: + - > + ./gradlew bundleDebugAar + -Dorg.gradle.project.buildDir=/libretroshare-build/ + -PANDROID_MIN_API_LEVEL=16 + -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" + -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" diff --git a/misc/Android/Dockerfile b/misc/Android/Dockerfile index ef2da812d..f93207b96 100644 --- a/misc/Android/Dockerfile +++ b/misc/Android/Dockerfile @@ -2,11 +2,14 @@ ## image name must match gitlab repository name, you can play just with the tag ## the part after : # export ANDROID_MIN_API_LEVEL=16 -# export CI_IMAGE_NAME="registry.gitlab.com/retroshare/retroshare:android_aar_base_$ANDROID_MIN_API_LEVEL" -# docker build --squash --tag "${CI_IMAGE_NAME}" --build-arg ANDROID_MIN_API_LEVEL=$ANDROID_MIN_API_LEVEL --build-arg LIBRETROSHARE_SOURCE_VERSION="$(git describe --always)" --file misc/Android/Dockerfile . +# export CI_IMAGE_NAME="registry.gitlab.com/retroshare/retroshare:android_aar_base_${ANDROID_MIN_API_LEVEL}" +# docker build --squash --tag "${CI_IMAGE_NAME}" --build-arg ANDROID_MIN_API_LEVEL=${ANDROID_MIN_API_LEVEL} --build-arg LIBRETROSHARE_SOURCE_VERSION="$(git describe --always)" --file misc/Android/Dockerfile . -## --squash is very important in case of GitlabCI shared runners as they are -## limited to 25GB disk size +## To fit GitlabCI shared runners as they are limited to 25GB disk size +## use --squash +## and set +## GRADLE_BUILD_TASK to bundleDebugAar or bundleReleaseAar +## JNI_NATIVE_LIBS_ARCHS to arm64-v8a or armeabi-v7a ## To push it to gitlab CI registry you need first to login and the to push # docker login registry.gitlab.com @@ -46,10 +49,14 @@ RUN /libretroshare/misc/Android/prepare-toolchain-clang.sh install_android_sdk WORKDIR /libretroshare/ ARG ANDROID_MIN_API_LEVEL=16 ARG LIBRETROSHARE_SOURCE_VERSION="Dockerfile_unset" -ENV NATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" -RUN ./gradlew build -Dorg.gradle.project.buildDir=/libretroshare-build/ \ +ARG GRADLE_BUILD_TASK="build" +ARG JNI_NATIVE_LIBS_ARCHS="arm64-v8a armeabi-v7a" +ARG NATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" +RUN ./gradlew $GRADLE_BUILD_TASK \ + -Dorg.gradle.project.buildDir=/libretroshare-build/ \ -PANDROID_MIN_API_LEVEL=$ANDROID_MIN_API_LEVEL \ - -PNATIVE_TOOLCHAINS_DIR=$NATIVE_TOOLCHAINS_DIR \ + -PNATIVE_TOOLCHAINS_DIR="$NATIVE_TOOLCHAINS_DIR" \ + -PJNI_NATIVE_LIBS_ARCHS="$JNI_NATIVE_LIBS_ARCHS" \ -PLIBRETROSHARE_SOURCE_VERSION="$LIBRETROSHARE_SOURCE_VERSION" WORKDIR / From 7a71fc8ea892e855c130b2d23f1825c3c12c381b Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Tue, 5 Apr 2022 11:17:42 +0200 Subject: [PATCH 07/21] Add artifacts path for shared runners job --- .gitlab-ci.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fec503aec..f53cbf9ce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,14 @@ +# Jobs tagged with `storage50` needs custom runners with more storage, they +# create a full build (multiple architecure, and both debug and release) a job +# for each minimum Android API level. After build the AAR packages are published +# on the project Gitlab maven repository + android-AAR-minapi16: rules: - if: $PROJECT_HAVE_STORAGE50_RUNNERS == "true" image: registry.gitlab.com/retroshare/retroshare:android_aar_base_16_build script: + - rm -rf /libretroshare-aar/ - > ./gradlew build publish -Dorg.gradle.project.buildDir=/libretroshare-build/ @@ -17,6 +23,7 @@ android-AAR-minapi21: - if: $PROJECT_HAVE_STORAGE50_RUNNERS == "true" image: registry.gitlab.com/retroshare/retroshare:android_aar_base_21 script: + - rm -rf /libretroshare-aar/ - > ./gradlew build publish -Dorg.gradle.project.buildDir=/libretroshare-build/ @@ -31,6 +38,7 @@ android-AAR-minapi24: - if: $PROJECT_HAVE_STORAGE50_RUNNERS == "true" image: registry.gitlab.com/retroshare/retroshare:android_aar_base_24 script: + - rm -rf /libretroshare-aar/ - > ./gradlew build publish -Dorg.gradle.project.buildDir=/libretroshare-build/ @@ -40,14 +48,24 @@ android-AAR-minapi24: tags: - storage50 + +# For forks that doesn't have `storage50` custom runners available the job are +# split per architecture and build type and without maven publishing, instead at +# the end of the job the artifact is made available + android-minapi16_armeabi-v7a_bundleDebugAar: rules: - if: $PROJECT_HAVE_STORAGE50_RUNNERS != "true" image: registry.gitlab.com/retroshare/retroshare:android_aar_base_16_armeabi-v7a_bundleDebugAar script: + - rm -rf /libretroshare-aar/ - > ./gradlew bundleDebugAar -Dorg.gradle.project.buildDir=/libretroshare-build/ -PANDROID_MIN_API_LEVEL=16 -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ + artifacts: + paths: + - outputs-aar/ From a77beda3cd6a99dd4df8049091521a18b6927d0a Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Tue, 5 Apr 2022 16:04:43 +0200 Subject: [PATCH 08/21] Add shared runners CI for more build types --- .gitlab-ci.yml | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f53cbf9ce..f0046b54e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -69,3 +69,54 @@ android-minapi16_armeabi-v7a_bundleDebugAar: artifacts: paths: - outputs-aar/ + +android-minapi16_armeabi-v7a_bundleReleaseAar: + rules: + - if: $PROJECT_HAVE_STORAGE50_RUNNERS != "true" + image: registry.gitlab.com/retroshare/retroshare:android_aar_base_16_armeabi-v7a_bundleReleaseAar + script: + - rm -rf /libretroshare-aar/ + - > + ./gradlew bundleReleaseAar + -Dorg.gradle.project.buildDir=/libretroshare-build/ + -PANDROID_MIN_API_LEVEL=16 + -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" + -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ + artifacts: + paths: + - outputs-aar/ + +android-minapi21_arm64-v8a_bundleDebugAar: + rules: + - if: $PROJECT_HAVE_STORAGE50_RUNNERS != "true" + image: registry.gitlab.com/retroshare/retroshare:android_aar_base_21_arm64-v8a_bundleDebugAar + script: + - rm -rf /libretroshare-aar/ + - > + ./gradlew bundleDebugAar + -Dorg.gradle.project.buildDir=/libretroshare-build/ + -PANDROID_MIN_API_LEVEL=21 + -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" + -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ + artifacts: + paths: + - outputs-aar/ + +android-minapi21_arm64-v8a_bundleReleaseAar: + rules: + - if: $PROJECT_HAVE_STORAGE50_RUNNERS != "true" + image: registry.gitlab.com/retroshare/retroshare:android_aar_base_21_arm64-v8a_bundleReleaseAar + script: + - rm -rf /libretroshare-aar/ + - > + ./gradlew bundleReleaseAar + -Dorg.gradle.project.buildDir=/libretroshare-build/ + -PANDROID_MIN_API_LEVEL=21 + -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" + -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ + artifacts: + paths: + - outputs-aar/ From 009972f644615eda8aef108c707e83c49b3b0950 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Tue, 5 Apr 2022 18:02:58 +0200 Subject: [PATCH 09/21] Add Android API 21 armeabi jobs for shared runners --- .gitlab-ci.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f0046b54e..385f12d65 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -120,3 +120,37 @@ android-minapi21_arm64-v8a_bundleReleaseAar: artifacts: paths: - outputs-aar/ + +android-minapi21_armeabi-v7a_bundleDebugAar: + rules: + - if: $PROJECT_HAVE_STORAGE50_RUNNERS != "true" + image: registry.gitlab.com/retroshare/retroshare:android_aar_base_21_armeabi-v7a_bundleDebugAar + script: + - rm -rf /libretroshare-aar/ + - > + ./gradlew bundleDebugAar + -Dorg.gradle.project.buildDir=/libretroshare-build/ + -PANDROID_MIN_API_LEVEL=21 + -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" + -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ + artifacts: + paths: + - outputs-aar/ + +android-minapi21_armeabi-v7a_bundleReleaseAar: + rules: + - if: $PROJECT_HAVE_STORAGE50_RUNNERS != "true" + image: registry.gitlab.com/retroshare/retroshare:android_aar_base_21_armeabi-v7a_bundleReleaseAar + script: + - rm -rf /libretroshare-aar/ + - > + ./gradlew bundleReleaseAar + -Dorg.gradle.project.buildDir=/libretroshare-build/ + -PANDROID_MIN_API_LEVEL=21 + -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" + -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ + artifacts: + paths: + - outputs-aar/ From 7e0bafc180f5b948766b8882310ecff06c67bc5e Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Thu, 7 Apr 2022 10:34:43 +0200 Subject: [PATCH 10/21] Android strip debug symbols in release, while keep them in debug build --- build.gradle | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/build.gradle b/build.gradle index f762fc45a..c7f16d61c 100644 --- a/build.gradle +++ b/build.gradle @@ -155,6 +155,23 @@ ext.copyChecked = assert file("$destDir/${file(sourceFile).getName()}").exists() } +ext.stripLib = +{ + toolchainPath, + libDir, + libFile -> + + def stripCmd = "$toolchainPath/bin/llvm-strip" + assert file(stripCmd).exists() + assert file("${libDir}/${libFile}").exists() + + exec + { + workingDir libDir + commandLine stripCmd, '--strip-unneeded', libFile + } +} + ext.buildLibretroshareNativeLib = { pApiLevel, /* Android API level */ @@ -235,6 +252,16 @@ ext.buildLibretroshareNativeLib = "${currToolchainPath}/sysroot/usr/lib/${libcxxsharedTriple}/${pApiLevel}/liblog.so", currAbiLibDir ) + /* Work around Android gradle stripping bug read more information near + * android.buildTypes.debug packagingOptions.jniLibs.keepDebugSymbols + * section of this file */ + if(pBuildType.equalsIgnoreCase("Release")) + { + stripLib(currToolchainPath, currAbiLibDir, "libretroshare.so") + stripLib(currToolchainPath, currAbiLibDir, "libc++_shared.so") + stripLib(currToolchainPath, currAbiLibDir, "liblog.so") + } + def bdbootAssetDir = "${getAssetsDir()}/values" mkdir bdbootAssetDir copyChecked( @@ -261,6 +288,42 @@ android setProperty("archivesBaseName", getArtifactBaseName()) + buildTypes + { + debug + { + debuggable true + jniDebuggable true + + /* The following line was added as an attempt to avoid stripping of + * debugging symbols, for debug build type + * https://developer.android.com/reference/tools/gradle-api/7.3/com/android/build/api/dsl/PackagingOptions#doNotStrip%28kotlin.String%29= + * https://developer.android.com/reference/tools/gradle-api/7.3/com/android/build/api/dsl/JniLibsPackagingOptions + * but it ends up affecting all other build types too. + * So enabling this without other countermeasures bloated release + * library with debugging symbols. + * Even more strange is that this is an old known bug at Google + * https://issuetracker.google.com/issues/155215248 + * https://stackoverflow.com/questions/52972371/set-donotstrip-packagingoptions-to-a-specific-buildtype + * but nothing substantial has been done to fix this even on newer + * versions of Android Gradle plugin. + * To work around this bug the release .so libraries must be + * stripped in advance in buildLibretroshareNativeLib + */ + packagingOptions.jniLibs.keepDebugSymbols += "**/*.so" + } + + release + { + /* The following was added as an attempt to ship separated debugging + * symbols in release mode, but had no effect last time I tested + * 2022/04/06 + * https://developer.android.com/studio/build/shrink-code#android_gradle_plugin_version_41_or_later + */ + //ndk.debugSymbolLevel 'FULL' + } + } + buildTypes.all { buildTypeObj -> From ce99b41bb2b8467541491f705b593eefd60e1457 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Thu, 7 Apr 2022 10:51:00 +0200 Subject: [PATCH 11/21] Add Android API 24 armeabi jobs for shared runners --- .gitlab-ci.yml | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 385f12d65..8fcf2670d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -154,3 +154,71 @@ android-minapi21_armeabi-v7a_bundleReleaseAar: artifacts: paths: - outputs-aar/ + +android-minapi24_arm64-v8a_bundleDebugAar: + rules: + - if: $PROJECT_HAVE_STORAGE50_RUNNERS != "true" + image: registry.gitlab.com/retroshare/retroshare:android_aar_base_24_arm64-v8a_bundleDebugAar + script: + - rm -rf /libretroshare-aar/ + - > + ./gradlew bundleDebugAar + -Dorg.gradle.project.buildDir=/libretroshare-build/ + -PANDROID_MIN_API_LEVEL=24 + -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" + -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ + artifacts: + paths: + - outputs-aar/ + +android-minapi24_arm64-v8a_bundleReleaseAar: + rules: + - if: $PROJECT_HAVE_STORAGE50_RUNNERS != "true" + image: registry.gitlab.com/retroshare/retroshare:android_aar_base_24_arm64-v8a_bundleReleaseAar + script: + - rm -rf /libretroshare-aar/ + - > + ./gradlew bundleReleaseAar + -Dorg.gradle.project.buildDir=/libretroshare-build/ + -PANDROID_MIN_API_LEVEL=24 + -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" + -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ + artifacts: + paths: + - outputs-aar/ + +android-minapi24_armeabi-v7a_bundleDebugAar: + rules: + - if: $PROJECT_HAVE_STORAGE50_RUNNERS != "true" + image: registry.gitlab.com/retroshare/retroshare:android_aar_base_24_armeabi-v7a_bundleDebugAar + script: + - rm -rf /libretroshare-aar/ + - > + ./gradlew bundleDebugAar + -Dorg.gradle.project.buildDir=/libretroshare-build/ + -PANDROID_MIN_API_LEVEL=24 + -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" + -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ + artifacts: + paths: + - outputs-aar/ + +android-minapi24_armeabi-v7a_bundleReleaseAar: + rules: + - if: $PROJECT_HAVE_STORAGE50_RUNNERS != "true" + image: registry.gitlab.com/retroshare/retroshare:android_aar_base_24_armeabi-v7a_bundleReleaseAar + script: + - rm -rf /libretroshare-aar/ + - > + ./gradlew bundleReleaseAar + -Dorg.gradle.project.buildDir=/libretroshare-build/ + -PANDROID_MIN_API_LEVEL=24 + -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" + -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ + artifacts: + paths: + - outputs-aar/ From 450027bfa296d98cc6f9342edfffc0c814a1029c Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Thu, 7 Apr 2022 10:58:24 +0200 Subject: [PATCH 12/21] Add JSON API to determine config directory --- src/retroshare/rsinit.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/retroshare/rsinit.h b/src/retroshare/rsinit.h index ba2c33765..2df5eb4ad 100644 --- a/src/retroshare/rsinit.h +++ b/src/retroshare/rsinit.h @@ -251,6 +251,8 @@ public: * @brief ConfigDirectory (usually ~/.retroshare) you can call this method * even before initialisation (you can't with some other methods) * @see RsAccountsDetail::PathBaseDirectory() + * @jsonapi{development,unauthenticated} + * @return a string containing the path */ static std::string ConfigDirectory(); From 116b3045c03063879553275dcf70cc9abf0d19a4 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Thu, 7 Apr 2022 11:00:27 +0200 Subject: [PATCH 13/21] Add script to prepare GitalbCI Android base images --- misc/Android/GitlabCI_images_prepare.sh | 79 +++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100755 misc/Android/GitlabCI_images_prepare.sh diff --git a/misc/Android/GitlabCI_images_prepare.sh b/misc/Android/GitlabCI_images_prepare.sh new file mode 100755 index 000000000..9457b2f3a --- /dev/null +++ b/misc/Android/GitlabCI_images_prepare.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# Script to prepare libretroshare Android testing and publishing Docker images +# +# Copyright (C) 2016-2022 Gioacchino Mazzurco +# Copyright (C) 2020-2022 Asociación Civil Altermundi +# +# This program is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, version 3. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License along +# with this program. If not, see +# +# SPDX-FileCopyrightText: Retroshare Team +# SPDX-License-Identifier: AGPL-3.0-only + +set -e + +## Define default value for variable, take two arguments, $1 variable name, +## $2 default variable value, if the variable is not already define define it +## with default value. +function define_default_value() +{ + VAR_NAME="${1}" + DEFAULT_VALUE="${2}" + + [ -z "${!VAR_NAME}" ] && export ${VAR_NAME}="${DEFAULT_VALUE}" || true +} + +define_default_value LIBRETROSHARE_SOURCE_VERSION "$(git describe --always)" +define_default_value ANDROID_MIN_API_LEVELS "16 21 24" +define_default_value JNI_NATIVE_LIBS_ARCHS "arm64-v8a armeabi-v7a" +define_default_value GRADLE_FULL_TASK "build" # set to "none" to disable +define_default_value GRADLE_SPLIT_TASKS "bundleDebugAar bundleReleaseAar" + + +function buildDetachPushAndClean() +{ + IMAGE_NAME="$1" + + docker build --squash --tag "${IMAGE_NAME}" \ + --build-arg ANDROID_MIN_API_LEVEL=${ANDROID_MIN_API_LEVEL} \ + --build-arg LIBRETROSHARE_SOURCE_VERSION="$LIBRETROSHARE_SOURCE_VERSION" \ + --build-arg GRADLE_BUILD_TASK="$GRADLE_BUILD_TASK" \ + --build-arg JNI_NATIVE_LIBS_ARCHS="$JNI_NATIVE_LIBS_ARCHS" \ + --file misc/Android/Dockerfile . + + # Push and clean in parallel with other builds + ((docker push "$IMAGE_NAME" && docker system prune --force)&) +} + +for mApiLevel in $(shuf --echo $ANDROID_MIN_API_LEVELS) ; do + ANDROID_MIN_API_LEVEL=$mApiLevel + [ "$mApiLevel" -gt "16" ] || JNI_NATIVE_LIBS_ARCHS="armeabi-v7a" + + [ "$GRADLE_FULL_TASK" == "none" ] || + { + GRADLE_BUILD_TASK="$GRADLE_FULL_TASK" + CI_IMAGE_NAME="registry.gitlab.com/retroshare/retroshare:android_aar_base_${ANDROID_MIN_API_LEVEL}" + + buildDetachPushAndClean "$CI_IMAGE_NAME" + } + + for mArch in $(shuf --echo $JNI_NATIVE_LIBS_ARCHS) ; do + for mTask in $(shuf --echo $GRADLE_SPLIT_TASKS); do + GRADLE_BUILD_TASK=$mTask + JNI_NATIVE_LIBS_ARCHS=$mArch + CI_IMAGE_NAME="registry.gitlab.com/retroshare/retroshare:android_aar_base_${ANDROID_MIN_API_LEVEL}_${JNI_NATIVE_LIBS_ARCHS}_${GRADLE_BUILD_TASK}" + + buildDetachPushAndClean "$CI_IMAGE_NAME" + done + done +done From e2ec6b9edc90b2e8ed7b7245690ad934af491c6d Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Thu, 7 Apr 2022 13:34:45 +0200 Subject: [PATCH 14/21] Android GitlabCI specify JNI arch in arch specific jobs --- .gitlab-ci.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8fcf2670d..5ee7b50bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,6 +65,7 @@ android-minapi16_armeabi-v7a_bundleDebugAar: -PANDROID_MIN_API_LEVEL=16 -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + -PJNI_NATIVE_LIBS_ARCHS="armeabi-v7a" - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ artifacts: paths: @@ -82,6 +83,7 @@ android-minapi16_armeabi-v7a_bundleReleaseAar: -PANDROID_MIN_API_LEVEL=16 -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + -PJNI_NATIVE_LIBS_ARCHS="armeabi-v7a" - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ artifacts: paths: @@ -99,6 +101,7 @@ android-minapi21_arm64-v8a_bundleDebugAar: -PANDROID_MIN_API_LEVEL=21 -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + -PJNI_NATIVE_LIBS_ARCHS="arm64-v8a" - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ artifacts: paths: @@ -116,6 +119,7 @@ android-minapi21_arm64-v8a_bundleReleaseAar: -PANDROID_MIN_API_LEVEL=21 -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + -PJNI_NATIVE_LIBS_ARCHS="arm64-v8a" - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ artifacts: paths: @@ -133,6 +137,7 @@ android-minapi21_armeabi-v7a_bundleDebugAar: -PANDROID_MIN_API_LEVEL=21 -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + -PJNI_NATIVE_LIBS_ARCHS="armeabi-v7a" - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ artifacts: paths: @@ -150,6 +155,7 @@ android-minapi21_armeabi-v7a_bundleReleaseAar: -PANDROID_MIN_API_LEVEL=21 -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + -PJNI_NATIVE_LIBS_ARCHS="armeabi-v7a" - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ artifacts: paths: @@ -167,6 +173,7 @@ android-minapi24_arm64-v8a_bundleDebugAar: -PANDROID_MIN_API_LEVEL=24 -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + -PJNI_NATIVE_LIBS_ARCHS="arm64-v8a" - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ artifacts: paths: @@ -184,6 +191,7 @@ android-minapi24_arm64-v8a_bundleReleaseAar: -PANDROID_MIN_API_LEVEL=24 -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + -PJNI_NATIVE_LIBS_ARCHS="arm64-v8a" - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ artifacts: paths: @@ -201,6 +209,7 @@ android-minapi24_armeabi-v7a_bundleDebugAar: -PANDROID_MIN_API_LEVEL=24 -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + -PJNI_NATIVE_LIBS_ARCHS="armeabi-v7a" - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ artifacts: paths: @@ -218,6 +227,7 @@ android-minapi24_armeabi-v7a_bundleReleaseAar: -PANDROID_MIN_API_LEVEL=24 -PLIBRETROSHARE_SOURCE_VERSION="$CI_COMMIT_SHORT_SHA" -PNATIVE_TOOLCHAINS_DIR="/libretroshare-native-toolchains/" + -PJNI_NATIVE_LIBS_ARCHS="armeabi-v7a" - mv /libretroshare-build/outputs/aar/ $CI_PROJECT_DIR/outputs-aar/ artifacts: paths: From cc7e70af966b7c42a05bde4490038b927fc9c188 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Thu, 7 Apr 2022 13:52:51 +0200 Subject: [PATCH 15/21] Fix GitlabCI regression introduced in f0a88b3d4d3bb46a51416cfdf2dc9f9977cb77b2 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5ee7b50bf..0154c8272 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,7 @@ android-AAR-minapi16: rules: - if: $PROJECT_HAVE_STORAGE50_RUNNERS == "true" - image: registry.gitlab.com/retroshare/retroshare:android_aar_base_16_build + image: registry.gitlab.com/retroshare/retroshare:android_aar_base_16 script: - rm -rf /libretroshare-aar/ - > From 56b469a15fb1f71c746bdddb9a0616c15f1df2c1 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Thu, 7 Apr 2022 11:02:05 +0200 Subject: [PATCH 16/21] Initial optimizations for Android release build --- CMakeLists.txt | 9 +++++++++ misc/Android/prepare-toolchain-clang.sh | 13 ++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e6a4cd135..5281b537b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -602,3 +602,12 @@ if(V_VERSION_SET) endif(V_VERSION_SET) ################################################################################ + +## On Android all symbols except for the few ones used called via JNI can be +## kept hidden allowing for more optimization, and reduced binary size +if(RS_ANDROID AND CMAKE_BUILD_TYPE STREQUAL "Release") + set_property(TARGET ${PROJECT_NAME} PROPERTY C_VISIBILITY_PRESET hidden) + set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_VISIBILITY_PRESET hidden) + set_property(TARGET ${PROJECT_NAME} PROPERTY VISIBILITY_INLINES_HIDDEN ON) + set_property(TARGET ${PROJECT_NAME} PROPERTY INTERPROCEDURAL_OPTIMIZATION ON) +endif(RS_ANDROID) diff --git a/misc/Android/prepare-toolchain-clang.sh b/misc/Android/prepare-toolchain-clang.sh index 7e5dbf8a5..c594a5803 100755 --- a/misc/Android/prepare-toolchain-clang.sh +++ b/misc/Android/prepare-toolchain-clang.sh @@ -139,6 +139,7 @@ export CC="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang export CXX="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang++" export AR="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ar" export RANLIB="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ranlib" +# More interesting GNU Make variables at http://www.gnu.org/software/make/manual/make.html#Implicit-Variables # Used to instruct cmake to explicitely ignore host libraries export HOST_IGNORE_PREFIX="/usr/" @@ -213,6 +214,12 @@ function andro_cmake() [ "$TOOLCHAIN_BUILD_TYPE" == "" ] || cmakeBuildType="-DCMAKE_BUILD_TYPE=$TOOLCHAIN_BUILD_TYPE" + cmakeOptimizationsOpt="" + [ "$TOOLCHAIN_BUILD_TYPE" != "Release" ] || + { + cmakeOptimizationsOpt="-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON" + } + cmake \ $cmakeBuildType \ -DCMAKE_SYSTEM_PROCESSOR=$cmakeProc \ @@ -828,9 +835,9 @@ build_libretroshare() -D RS_BRODCAST_DISCOVERY=ON -D RS_EXPORT_JNI_ONLOAD=ON \ -D RS_SQLCIPHER=OFF -D RS_DH_PRIME_INIT_CHECK=OFF \ -D RS_FORUM_DEEP_INDEX=ON -D RS_JSON_API=ON \ - $RS_EXTRA_CMAKE_OPTS || return $? - make -j${HOST_NUM_CPU} || return $? - make install || return $? + $RS_EXTRA_CMAKE_OPTS + make -j${HOST_NUM_CPU} + make install popd )} From 3479ec6ba42e6826cf198401d0ca0dc15bf2d2c0 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Fri, 8 Apr 2022 11:35:35 +0200 Subject: [PATCH 17/21] Fix erratic Docker image script generator --- misc/Android/GitlabCI_images_prepare.sh | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/misc/Android/GitlabCI_images_prepare.sh b/misc/Android/GitlabCI_images_prepare.sh index 9457b2f3a..74a99adeb 100755 --- a/misc/Android/GitlabCI_images_prepare.sh +++ b/misc/Android/GitlabCI_images_prepare.sh @@ -40,7 +40,7 @@ define_default_value GRADLE_FULL_TASK "build" # set to "none" to disable define_default_value GRADLE_SPLIT_TASKS "bundleDebugAar bundleReleaseAar" -function buildDetachPushAndClean() +function buildDetachPush() { IMAGE_NAME="$1" @@ -48,32 +48,33 @@ function buildDetachPushAndClean() --build-arg ANDROID_MIN_API_LEVEL=${ANDROID_MIN_API_LEVEL} \ --build-arg LIBRETROSHARE_SOURCE_VERSION="$LIBRETROSHARE_SOURCE_VERSION" \ --build-arg GRADLE_BUILD_TASK="$GRADLE_BUILD_TASK" \ - --build-arg JNI_NATIVE_LIBS_ARCHS="$JNI_NATIVE_LIBS_ARCHS" \ + --build-arg JNI_NATIVE_LIBS_ARCHS="$CURR_JNI_NATIVE_LIBS_ARCHS" \ --file misc/Android/Dockerfile . - # Push and clean in parallel with other builds - ((docker push "$IMAGE_NAME" && docker system prune --force)&) + # Start pushing in parallel with other build + ((sleep 1m ; docker push "$IMAGE_NAME")&) } for mApiLevel in $(shuf --echo $ANDROID_MIN_API_LEVELS) ; do ANDROID_MIN_API_LEVEL=$mApiLevel - [ "$mApiLevel" -gt "16" ] || JNI_NATIVE_LIBS_ARCHS="armeabi-v7a" + CURR_JNI_NATIVE_LIBS_ARCHS="$JNI_NATIVE_LIBS_ARCHS" + [ "$mApiLevel" -gt "16" ] || CURR_JNI_NATIVE_LIBS_ARCHS="armeabi-v7a" [ "$GRADLE_FULL_TASK" == "none" ] || { GRADLE_BUILD_TASK="$GRADLE_FULL_TASK" CI_IMAGE_NAME="registry.gitlab.com/retroshare/retroshare:android_aar_base_${ANDROID_MIN_API_LEVEL}" - buildDetachPushAndClean "$CI_IMAGE_NAME" + buildDetachPush "$CI_IMAGE_NAME" } - for mArch in $(shuf --echo $JNI_NATIVE_LIBS_ARCHS) ; do + for mArch in $(shuf --echo $CURR_JNI_NATIVE_LIBS_ARCHS) ; do for mTask in $(shuf --echo $GRADLE_SPLIT_TASKS); do GRADLE_BUILD_TASK=$mTask - JNI_NATIVE_LIBS_ARCHS=$mArch - CI_IMAGE_NAME="registry.gitlab.com/retroshare/retroshare:android_aar_base_${ANDROID_MIN_API_LEVEL}_${JNI_NATIVE_LIBS_ARCHS}_${GRADLE_BUILD_TASK}" + CURR_JNI_NATIVE_LIBS_ARCHS=$mArch + CI_IMAGE_NAME="registry.gitlab.com/retroshare/retroshare:android_aar_base_${ANDROID_MIN_API_LEVEL}_${CURR_JNI_NATIVE_LIBS_ARCHS}_${GRADLE_BUILD_TASK}" - buildDetachPushAndClean "$CI_IMAGE_NAME" + buildDetachPush "$CI_IMAGE_NAME" done done done From db0a826027ed556b6ec0a6f96a8d3269b895ed31 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Fri, 8 Apr 2022 11:36:07 +0200 Subject: [PATCH 18/21] Android strip only debug symbols in system libs --- build.gradle | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/build.gradle b/build.gradle index c7f16d61c..938daa099 100644 --- a/build.gradle +++ b/build.gradle @@ -159,7 +159,8 @@ ext.stripLib = { toolchainPath, libDir, - libFile -> + libFile, + stripOptions = "" -> def stripCmd = "$toolchainPath/bin/llvm-strip" assert file(stripCmd).exists() @@ -168,7 +169,7 @@ ext.stripLib = exec { workingDir libDir - commandLine stripCmd, '--strip-unneeded', libFile + commandLine stripCmd, stripOptions, libFile } } @@ -257,9 +258,9 @@ ext.buildLibretroshareNativeLib = * section of this file */ if(pBuildType.equalsIgnoreCase("Release")) { - stripLib(currToolchainPath, currAbiLibDir, "libretroshare.so") - stripLib(currToolchainPath, currAbiLibDir, "libc++_shared.so") - stripLib(currToolchainPath, currAbiLibDir, "liblog.so") + stripLib(currToolchainPath, currAbiLibDir, "libretroshare.so", "--strip-unneeded") + stripLib(currToolchainPath, currAbiLibDir, "libc++_shared.so", "--strip-debug") + stripLib(currToolchainPath, currAbiLibDir, "liblog.so", "--strip-debug") } def bdbootAssetDir = "${getAssetsDir()}/values" From 3fb8f44bd28f415924495b225f25a80d3d95dced Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Fri, 8 Apr 2022 13:27:36 +0200 Subject: [PATCH 19/21] Android specify float-abi for ARMv7 --- misc/Android/prepare-toolchain-clang.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/misc/Android/prepare-toolchain-clang.sh b/misc/Android/prepare-toolchain-clang.sh index c594a5803..4c2a2aa56 100755 --- a/misc/Android/prepare-toolchain-clang.sh +++ b/misc/Android/prepare-toolchain-clang.sh @@ -111,7 +111,6 @@ define_default_value TOOLCHAIN_BUILD_TYPE "" cArch="" eABI="" -cmakeABI="" case "${ANDROID_NDK_ARCH}" in "arm") @@ -144,6 +143,7 @@ export RANLIB="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-r # Used to instruct cmake to explicitely ignore host libraries export HOST_IGNORE_PREFIX="/usr/" +export ARMv7_OPTIMIZATION_FLAGS="-march=armv7-a -mfloat-abi=softfp -mfpu=vfp" ## $1 filename, $2 sha256 hash function check_sha256() @@ -188,10 +188,13 @@ function andro_cmake() # https://developer.android.com/ndk/guides/cmake seens to break more things then # it fixes :-\ + cmakeArchFlags="" cmakeProc="" case "${ANDROID_NDK_ARCH}" in "arm") cmakeProc="armv7-a" + export CFLAGS="$ARMv7_OPTIMIZATION_FLAGS" + export CXXFLAGS="$ARMv7_OPTIMIZATION_FLAGS" ;; "arm64") cmakeProc="aarch64" @@ -436,10 +439,13 @@ build_openssl() ## non neglegible security concerns. oBits="32" [[ ${ANDROID_NDK_ARCH} =~ .*64.* ]] && oBits=64 + + armOptimizationFlags="" + [[ "${ANDROID_NDK_ARCH}" != "arm" ]] || armOptimizationFlags="$ARMv7_OPTIMIZATION_FLAGS" ANDROID_NDK="${ANDROID_NDK_PATH}" PATH="${SYSROOT}/bin/:${PATH}" \ - ./Configure linux-generic${oBits} -fPIC --prefix="${PREFIX}" \ - --openssldir="${SYSROOT}/etc/ssl" + ./Configure linux-generic${oBits} -fPIC $armOptimizationFlags \ + --prefix="${PREFIX}" --openssldir="${SYSROOT}/etc/ssl" # sed -i 's/LIBNAME=$$i LIBVERSION=$(SHLIB_MAJOR).$(SHLIB_MINOR) \\/LIBNAME=$$i \\/g' Makefile # sed -i '/LIBCOMPATVERSIONS=";$(SHLIB_VERSION_HISTORY)" \\/d' Makefile From a8cd534dfbd90c104168b51cddf0de8087420784 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Fri, 8 Apr 2022 13:28:03 +0200 Subject: [PATCH 20/21] Android workaround missing C++17 filesystem symbols Workaround for bug https://github.com/android/ndk/issues/609 --- CMakeLists.txt | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5281b537b..c1d30ef8b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -215,7 +215,21 @@ if(RS_LIBRETROSHARE_SHARED) set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif(RS_LIBRETROSHARE_SHARED) -set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 17) +if(RS_ANDROID) + # As of today Android NDK libc++ doesn't support std::filesystem properly, + # in a very confusing manner the headers have the function declarations but + # then the shared object file doesn't have the implementetion. + # This fools the compiler which doesn't report any error but then at runtime + # we get a crash with this error: + # java.lang.UnsatisfiedLinkError: dlopen failed: cannot locate symbol + # "_ZNSt6__ndk14__fs10filesystem20__create_directoriesERKNS1_4pathEPNS_10error_codeE" + # referenced by "/data/app/==/lib/arm64/libretroshare.so + # So set maximum supported C++ version to 14 to workaround this Android bug + # https://github.com/android/ndk/issues/609 + set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 14) +else() + set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 17) +endif() ## As of today libretroshare doesn't hide implementation details properly so it ## is necessary to flag all implementation headers as public From 51da6f86fde61c0fae51748dd090e4b9bd3bdccd Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Fri, 8 Apr 2022 16:48:54 +0200 Subject: [PATCH 21/21] More configurability for Docker image script generator --- misc/Android/GitlabCI_images_prepare.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/Android/GitlabCI_images_prepare.sh b/misc/Android/GitlabCI_images_prepare.sh index 74a99adeb..05b1916f9 100755 --- a/misc/Android/GitlabCI_images_prepare.sh +++ b/misc/Android/GitlabCI_images_prepare.sh @@ -68,6 +68,7 @@ for mApiLevel in $(shuf --echo $ANDROID_MIN_API_LEVELS) ; do buildDetachPush "$CI_IMAGE_NAME" } + [ "$GRADLE_SPLIT_TASKS" == "none" ] || for mArch in $(shuf --echo $CURR_JNI_NATIVE_LIBS_ARCHS) ; do for mTask in $(shuf --echo $GRADLE_SPLIT_TASKS); do GRADLE_BUILD_TASK=$mTask