[INFRA-1363][WIP] Rpm repository can generate metadata from staging and production packages directory

.gitignore

@@ -10,4 +10,5 @@ generated
 .vagrant
 .iml
 .idea/
-*.orig
+*.orig
+credentials/ssh/known_hosts

deb/publish/publish.sh

@@ -79,9 +79,14 @@ function init(){
 function skipIfAlreadyPublished(){
 
   if ssh "${SSH_OPTS[@]}" "$PKGSERVER" test -e "${DEBDIR}/$(basename "$DEB")"; then
-    echo "File already published, nothing else todo"
+    echo "File already published $PKGSERVER:$PROD_DEBDIR, nothing else todo"
     return 0
   fi
+
+  if ssh "${SSH_OPTS[@]}" "$PKGSERVER" test -e "${PROD_DEBDIR}/$(basename "$DEB")"; then
+    echo "File already published on $PKGSERVER:$PROD_DEBDIR, nothing else todo"
+    exit 0
+  fi
   return 1
 
 }

docker-compose.yaml

@@ -1,4 +1,3 @@
-# docker exec -i -t packaging_packaging_1 gpg --import --batch credentials/sandbox.gpg
 version: '3'
 volumes:
   sshd:
@@ -41,3 +40,14 @@ services:
     volumes:
       - "./pkgConfig/httpd.conf:/usr/local/apache2/conf/httpd.conf"
       - "./target:/usr/local/apache2/htdocs"
+
+## Howto
+# docker exec -i -t packaging_packaging_1 gpg --import --batch credentials/sandbox.gpg
+#
+# Delete .ssh/known_hosts
+# docker exec -i -t packaging_packaging_1 rm .ssh/known_hosts
+# docker exec -i -t packaging_packaging_1 ssh jenkins@remote
+# docker exec -i -t packaging_packaging_1 make rpm rpm.publish
+# docker exec -i -t packaging_packaging_1 make deb deb.publish
+# docker exec -i -t packaging_packaging_1 make suse suse.publish
+# docker exec -i -t packaging_packaging_1 make war war.publish

env/release.mk

@@ -11,10 +11,14 @@ export SSH_OPTS=-p 22
 export SCP_OPTS=-P 22
 
 # where to put binary files
-export WARDIR=/srv/releases/jenkins/war${RELEASELINE}
-export MSIDIR=/srv/releases/jenkins/windows${RELEASELINE}
-export OSXDIR=/srv/releases/jenkins/osx${RELEASELINE}
-export DEBDIR=/srv/releases/jenkins/debian${RELEASELINE}
+export WARDIR=/srv/releases/jenkins.staging/war${RELEASELINE}
+export MSIDIR=/srv/releases/jenkins.staging/windows${RELEASELINE}
+export OSXDIR=/srv/releases/jenkins.staging/osx${RELEASELINE}
+export DEBDIR=/srv/releases/jenkins.staging/debian${RELEASELINE}
+export RPMDIR=/srv/releases/jenkins.staging/redhat${RELEASELINE}
+export SUSEDIR=/srv/releases/jenkins.staging/opensuse${RELEASELINE}
+
+# Where to fetch production packages used to generate pkg website
 export RPMDIR=/srv/releases/jenkins/redhat${RELEASELINE}
 export SUSEDIR=/srv/releases/jenkins/opensuse${RELEASELINE}
 

env/test.mk

@@ -10,24 +10,28 @@ export PKGSERVER=jenkins@remote
 # Testing both with and without SSH_OPTS
 #export SSH_OPTS=-p 22
 #export SCP_OPTS=-P 22
-export SSH_OPTS=-p 22
-export SCP_OPTS=-P 22
+export SSH_OPTS=-p 22 -o StrictHostKeyChecking=no
+export SCP_OPTS=-P 22 -o StrictHostKeyChecking=no
 
 # where to put binary files
-export TESTDIR=$(realpath .)/pkg.jenkins.io
-export WARDIR=${TESTDIR}/war${RELEASELINE}
-export MSIDIR=${TESTDIR}/windows${RELEASELINE}
-export OSXDIR=${TESTDIR}/osx${RELEASELINE}
-export DEBDIR=${TESTDIR}/debian${RELEASELINE}/binary
-export RPMDIR=${TESTDIR}/redhat${RELEASELINE}
-export SUSEDIR=${TESTDIR}/opensuse${RELEASELINE}
+export TESTDIR=/src/releases/jenkins
+export WARDIR=${TESTDIR}.staging/war${RELEASELINE}
+export MSIDIR=${TESTDIR}.staging/windows${RELEASELINE}
+export OSXDIR=${TESTDIR}.staging/osx${RELEASELINE}
+export DEBDIR=${TESTDIR}.staging/debian${RELEASELINE}/binary
+export RPMDIR=${TESTDIR}.staging/redhat${RELEASELINE}
+export SUSEDIR=${TESTDIR}.staging/opensuse${RELEASELINE}
+
+export PROD_RPMDIR=${TESTDIR}/redhat${RELEASELINE}
+export PROD_SUSEDIR=${TESTDIR}/opensuse${RELEASELINE}
 
 # where to put repository index and other web contents
-export  RPM_WEBDIR=${TESTDIR}/redhat${RELEASELINE}
-export SUSE_WEBDIR=${TESTDIR}/opensuse${RELEASELINE}
-export  DEB_WEBDIR=${TESTDIR}/debian${RELEASELINE}
-export  WAR_WEBDIR=${TESTDIR}/war${RELEASELINE}
-export  MSI_WEBDIR=${TESTDIR}/windows${RELEASELINE}
+export TESTDIR=$(realpath .)/pkg.jenkins.io
+export  RPM_WEBDIR=${TESTDIR}.staging/redhat${RELEASELINE}
+export SUSE_WEBDIR=${TESTDIR}.staging/opensuse${RELEASELINE}
+export  DEB_WEBDIR=${TESTDIR}.staging/debian${RELEASELINE}
+export  WAR_WEBDIR=${TESTDIR}.staging/war${RELEASELINE}
+export  MSI_WEBDIR=${TESTDIR}.staging/windows${RELEASELINE}
 
 # URL to the aforementioned webdir.
 WEBSERVER=pkg.jenkins.io

msi/publish/publish.sh

@@ -38,9 +38,13 @@ function init(){
 function skipIfAlreadyPublished(){
 
   if ssh "${SSH_OPTS[@]}" "$PKGSERVER" test -e "${MSIDIR}/${VERSION}/$(basename "$MSI")"; then
-    echo "File already published, nothing else todo"
+    echo "File already published $PKGSERVER:${MSIDIR}/${VERSION}/$(basename "$MSI"), nothing else todo"
     exit 0
+  fi
 
+  if ssh "${SSH_OPTS[@]}" "$PKGSERVER" test -e "${PROD_MSIDIR}/${VERSION}/$(basename "$MSI")"; then
+    echo "File already published on $PKGSERVER:${PROD_MSIDIR}/${VERSION}/$(basename "$MSI"), nothing else todo"
+    exit 0
   fi
 }
 

rpm/publish/publish.sh

@@ -23,11 +23,11 @@ function generateSite(){
   "$BASE/bin/indexGenerator.py" \
     --distribution redhat \
     --targetDir "${D}"
-  
+
   gpg --export -a --output "$D/${ORGANIZATION}.key" "${GPG_KEYNAME}"
-  
+
   "$BASE/bin/branding.py" "$D"
-  
+
   cp "$RPM" "$D/RPMS/noarch"
 
 cat  > "$D/${ARTIFACTNAME}.repo" << EOF
@@ -37,25 +37,55 @@ baseurl=${RPM_URL}
 gpgcheck=1
 EOF
 
-  # generate index 
+  # generate index
   # locally
   # disable this for now, as it's currently now used and generate errors
   # createrepo --update -o "$RPM_WEBDIR" "$RPMDIR/"
   # on the server
   # shellcheck disable=SC2029
-  ssh "${SSH_OPTS[@]}" "$PKGSERVER"  createrepo --update -o "'$RPM_WEBDIR'" "'$RPMDIR/'"
+  # --update can't be used because of https://bugs.centos.org/view.php?id=9189
+
+  if ! isDuplicatedPackageLocation; then
+    ssh "${SSH_OPTS[@]}" "$PKGSERVER"  createrepo --outputdir "'$RPM_WEBDIR'" --update --pretty --baseurl "'$PROD_RPMDIR/'" "'$RPMDIR/'"
+  else
+    echo "Please first remove duplicated packages between $RPMDIR/ and $PROD_RPMDIR"
+  fi
 
 }
 
-function skipIfAlreadyPublished(){
+function isDuplicatedPackageLocation(){
 
-  if ssh "${SSH_OPTS[@]}" "$PKGSERVER" test -e "${RPMDIR}/$(basename "$RPM")"; then
-    echo "File already published, nothing else todo"
-    exit 0
+  DUPLICATED=false
+
+  while read -r FILE; do
+    FILE=$(basename "$FILE")
+    if ssh "${SSH_OPTS[@]}" "$PKGSERVER" "test -e $PROD_RPMDIR/$FILE"; then
+      DUPLICATED=true
+      echo "Duplicated file '$FILE' found in both $RPMDIR $PROD_RPMDIR"
+    fi
+  done < <(ssh "${SSH_OPTS[@]}" "$PKGSERVER" find "$RPMDIR" -name "*.rpm" -type f)
+
+  return $DUPLICATED
 
-  fi
 }
 
+function isDuplicatedPackageLocation(){
+
+  DUPLICATED=false
+
+  while read -r FILE; do
+    FILE=$(basename "$FILE")
+    if [ -f "$PROD_RPMDIR/$FILE" ]; then
+      echo "Duplicated file '$FILE' found in both $RPMDIR $PROD_RPMDIR"
+    fi
+  done < <(find "$RPMDIR" -name "*.rpm" -type f)
+
+
+  return $DUPLICATED
+
+}
+
+
 function init(){
   mkdir -p "$D/RPMS/noarch"
 
@@ -76,7 +106,7 @@ function uploadPackage(){
     --progress \
     "$RPM" "$RPMDIR/"
 
-  # Remote 
+  # Remote
   rsync \
     --archive \
     --verbose \

suse/publish/publish.sh

@@ -24,11 +24,11 @@ function generateSite(){
   "$BASE/bin/indexGenerator.py" \
     --distribution opensuse \
     --targetDir "${D}"
-  
+
   gpg --export -a --output "$D/repodata/repomd.xml.key" "${GPG_KEYNAME}"
-  
+
   "$BASE/bin/branding.py" $D
-  
+
   cp "$SUSE" $D/RPMS/noarch
 }
 
@@ -45,12 +45,31 @@ function init(){
   mkdir -p $D/RPMS/noarch $D/repodata
 }
 
+function isDuplicatedPackageLocation(){
+
+  DUPLICATED=false
+
+  while read -r FILE; do
+    FILE=$(basename "$FILE")
+    if ssh "${SSH_OPTS[@]}" "$PKGSERVER" "test -e $PROD_SUSEDIR/$FILE"; then
+      DUPLICATED=true
+      echo "Duplicated file '$FILE' found in both $SUSEDIR $PROD_SUSEDIR"
+    fi
+  done < <(ssh "${SSH_OPTS[@]}" "$PKGSERVER" find "$SUSEDIR" -name "*.rpm" -type f)
+
+  return $DUPLICATED
+
+}
+
 function skipIfAlreadyPublished(){
 
   if ssh "${SSH_OPTS[@]}" "$PKGSERVER" "test -e ${SUSEDIR}/$(basename $SUSE)"; then
-    echo "File already published, nothing else todo"
+    echo "File already published $PKGSERVER:${SUSEDIR}/$(basename "$SUSE"), nothing else todo"
+    exit 0
+  fi
+  if ssh "${SSH_OPTS[@]}" "$PKGSERVER" "test -e ${PROD_SUSEDIR}/$(basename $SUSE)"; then
+    echo "File already published $PKGSERVER:${PROD_SUSEDIR}/$(basename "$SUSE"), nothing else todo"
     exit 0
-
   fi
 
 }
@@ -109,15 +128,21 @@ function uploadSite(){
       --exclude "HEADER.html" \
       --exclude "FOOTER.html" \
       . "$PKGSERVER:${SUSE_WEBDIR// /\\ }/" # Remote
-  
+
     # generate index on the server
     # server needs 'createrepo' pacakge
     # Disable this for now as not critical
     # createrepo --update -o "$SUSE_WEBDIR" "$SUSEDIR/" #Local
     # cp "${SUSE_WEBDIR// /\\ }/repodata/repomd.xml" repodata/ # Local
 
     # shellcheck disable=SC2029
-    ssh "${SSH_OPTS[@]}" "$PKGSERVER"   createrepo --update -o "'$SUSE_WEBDIR'" "'$SUSEDIR/'" # Remote
+    # --update can't be used because of https://bugs.centos.org/view.php?id=9189
+
+    if ! isDuplicatedPackageLocation; then
+      ssh "${SSH_OPTS[@]}" "$PKGSERVER"   createrepo --outputdir "'$SUSE_WEBDIR'" --pretty --split "'$PROD_SUSEDIR/'"  "'$SUSEDIR/'" # Remote
+    else
+      echo "Please first remove duplicated packages between $RPMDIR/ and $PROD_RPMDIR"
+    fi
 
     scp \
       "${SCP_OPTS[@]}" \
@@ -163,7 +188,7 @@ function uploadSite(){
       --exclude "*" \
       --progress \
       . "$PKGSERVER:${SUSEDIR// /\\ }/"
-    
+
   popd
 }
 

war/publish/publish.sh

@@ -39,9 +39,12 @@ function init(){
 function skipIfAlreadyPublished(){
 
   if ssh "${SSH_OPTS[@]}" "$PKGSERVER" test -e "${WARDIR}/${VERSION}/${ARTIFACTNAME}.war"; then
-    echo "File already published, nothing else todo"
+    echo "File already published $PKGSERVER:${WARDIR}/${VERSION}/${ARTIFACTNAME}.war, nothing else todo"
+    exit 0
+  fi
+  if ssh "${SSH_OPTS[@]}" "$PKGSERVER" test -e "${PROD_WARDIR}/${VERSION}/${ARTIFACTNAME}.war"; then
+    echo "File already published $PKGSERVER:${PROD_WARDIR}/${VERSION}/${ARTIFACTNAME}.war, nothing else todo"
     exit 0
-
   fi
 }