diff --git a/src/basic/cgroup-util.c b/src/basic/cgroup-util.c
index 6b84cc48c6..5de64d990e 100644
--- a/src/basic/cgroup-util.c
+++ b/src/basic/cgroup-util.c
@@ -30,6 +30,7 @@
 #include "string-table.h"
 #include "string-util.h"
 #include "strv.h"
+#include "unaligned.h"
 #include "unit-name.h"
 #include "user-util.h"
 #include "xattr-util.h"
@@ -38,7 +39,7 @@
 typedef union {
         struct file_handle file_handle;
         uint8_t space[offsetof(struct file_handle, f_handle) + sizeof(uint64_t)];
-} _alignas_(uint64_t) cg_file_handle;
+} cg_file_handle;
 
 #define CG_FILE_HANDLE_INIT                                     \
         (cg_file_handle) {                                      \
@@ -46,7 +47,8 @@ typedef union {
                 .file_handle.handle_type = FILEID_KERNFS,       \
         }
 
-#define CG_FILE_HANDLE_CGROUPID(fh) (*CAST_ALIGN_PTR(uint64_t, (fh).file_handle.f_handle))
+/* The .f_handle field is not aligned to 64bit on some archs, hence read it via an unaligned accessor */
+#define CG_FILE_HANDLE_CGROUPID(fh) unaligned_read_ne64(fh.file_handle.f_handle)
 
 int cg_path_open(const char *controller, const char *path) {
         _cleanup_free_ char *fs = NULL;
@@ -71,7 +73,7 @@ int cg_cgroupid_open(int cgroupfs_fd, uint64_t id) {
         }
 
         cg_file_handle fh = CG_FILE_HANDLE_INIT;
-        CG_FILE_HANDLE_CGROUPID(fh) = id;
+        unaligned_write_ne64(fh.file_handle.f_handle, id);
 
         return RET_NERRNO(open_by_handle_at(cgroupfs_fd, &fh.file_handle, O_DIRECTORY|O_CLOEXEC));
 }
diff --git a/src/basic/pidfd-util.c b/src/basic/pidfd-util.c
index 328ed70d52..9910819aa7 100644
--- a/src/basic/pidfd-util.c
+++ b/src/basic/pidfd-util.c
@@ -16,6 +16,7 @@
 #include "stat-util.h"
 #include "stdio-util.h"
 #include "string-util.h"
+#include "unaligned.h"
 
 static thread_local int have_pidfs = -1;
 
@@ -239,7 +240,7 @@ int pidfd_get_inode_id_impl(int fd, uint64_t *ret) {
                 union {
                         struct file_handle file_handle;
                         uint8_t space[offsetof(struct file_handle, f_handle) + sizeof(uint64_t)];
-                } _alignas_(uint64_t) fh = {
+                } fh = {
                         .file_handle.handle_bytes = sizeof(uint64_t),
                         .file_handle.handle_type = FILEID_KERNFS,
                 };
@@ -248,7 +249,8 @@ int pidfd_get_inode_id_impl(int fd, uint64_t *ret) {
                 r = RET_NERRNO(name_to_handle_at(fd, "", &fh.file_handle, &mnt_id, AT_EMPTY_PATH));
                 if (r >= 0) {
                         if (ret)
-                                *ret = *CAST_ALIGN_PTR(uint64_t, fh.file_handle.f_handle);
+                                /* Note, "struct file_handle" is 32bit aligned usually, but we need to read a 64bit value from it */
+                                *ret = unaligned_read_ne64(fh.file_handle.f_handle);
                         return 0;
                 }
                 assert(r != -EOVERFLOW);