diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp
index c91f412b84..9e74b5ea38 100644
--- a/src/video_core/gpu.cpp
+++ b/src/video_core/gpu.cpp
@@ -34,7 +34,6 @@ GPU::GPU(VideoCore::RasterizerInterface& rasterizer) {
     maxwell_compute = std::make_unique<Engines::MaxwellCompute>();
     maxwell_dma = std::make_unique<Engines::MaxwellDMA>(rasterizer, *memory_manager);
     kepler_memory = std::make_unique<Engines::KeplerMemory>(rasterizer, *memory_manager);
-    regs.semaphore_off_val = true;
 }
 
 GPU::~GPU() = default;
@@ -267,8 +266,7 @@ void GPU::ProcessBindMethod(const MethodCall& method_call) {
 
 void GPU::ProcessSemaphoreTriggerMethod() {
     const auto semaphoreOperationMask = 0xF;
-    const auto sequence = regs.reg_array[static_cast<u32>(BufferMethods::SemaphoreTrigger)];
-    const auto op = static_cast<GpuSemaphoreOperation>(sequence & semaphoreOperationMask);
+    const auto op = static_cast<GpuSemaphoreOperation>(regs.semaphore_trigger & semaphoreOperationMask);
     if (op == GpuSemaphoreOperation::WriteLong) {
         auto address = memory_manager->GpuToCpuAddress(regs.smaphore_address.SmaphoreAddress());
         struct Block {
@@ -284,7 +282,9 @@ void GPU::ProcessSemaphoreTriggerMethod() {
         block.timestamp = CoreTiming::GetTicks();
         Memory::WriteBlock(*address, &block, sizeof(block));
     } else {
-        const u32 word = Memory::Read32(regs.smaphore_address.SmaphoreAddress());
+        const auto address =
+            memory_manager->GpuToCpuAddress(regs.smaphore_address.SmaphoreAddress());
+        const u32 word = Memory::Read32(*address);
         if ((op == GpuSemaphoreOperation::AcquireEqual && word == regs.semaphore_sequence) ||
             (op == GpuSemaphoreOperation::AcquireGequal &&
              static_cast<s32>(word - regs.semaphore_sequence) > 0) ||
@@ -311,21 +311,14 @@ void GPU::ProcessSemaphoreTriggerMethod() {
 }
 
 void GPU::ProcessSemaphoreRelease() {
-    if (!regs.semaphore_off_val) {
-        LOG_ERROR(HW_GPU, "Semaphore can't be released since it is not currently been acquired");
-        return;
-    }
-    Memory::Write32(regs.smaphore_address.SmaphoreAddress(),
-                    regs.reg_array[static_cast<u32>(BufferMethods::SemaphoreRelease)]);
+    const auto address = memory_manager->GpuToCpuAddress(regs.smaphore_address.SmaphoreAddress());
+    Memory::Write32(*address, regs.semaphore_release);
 }
 
 void GPU::ProcessSemaphoreAcquire() {
-    if (!regs.semaphore_off_val) {
-        LOG_ERROR(HW_GPU, "Semaphore has already be acquired");
-        return;
-    }
-    const u32 word = Memory::Read32(regs.smaphore_address.SmaphoreAddress());
-    const auto value = regs.reg_array[static_cast<u32>(BufferMethods::SemaphoreAcquire)];
+    const auto address = memory_manager->GpuToCpuAddress(regs.smaphore_address.SmaphoreAddress());
+    const u32 word = Memory::Read32(*address);
+    const auto value = regs.semaphore_acquire;
     if (word != value) {
         regs.acquire_active = true;
         regs.acquire_value = value;
diff --git a/src/video_core/gpu.h b/src/video_core/gpu.h
index d277a3f9df..fb89758114 100644
--- a/src/video_core/gpu.h
+++ b/src/video_core/gpu.h
@@ -157,31 +157,40 @@ public:
     const Tegra::DmaPusher& DmaPusher() const;
 
     struct Regs {
-        static constexpr size_t NUM_REGS = 0x140;
+        static constexpr size_t NUM_REGS = 0x100;
 
         union {
             struct {
-                u32 acquire_mode;
-                u32 acquire_source;
+                INSERT_PADDING_WORDS(0x4);
+                struct {
+                    u32 address_high;
+                    u32 address_low;
+
+                    GPUVAddr SmaphoreAddress() const {
+                        return static_cast<GPUVAddr>((static_cast<GPUVAddr>(address_high) << 32) |
+                                                     address_low);
+                    }
+                } smaphore_address;
+
+                u32 semaphore_sequence;
+                u32 semaphore_trigger;
+                INSERT_PADDING_WORDS(0xC);
+
                 // The puser and the puller share the reference counter, the pusher only has read
                 // access
                 u32 reference_count;
+                INSERT_PADDING_WORDS(0x5);
+
+                u32 semaphore_acquire;
+                u32 semaphore_release;
+                INSERT_PADDING_WORDS(0xE4);
+
+                // Puller state
+                u32 acquire_mode;
+                u32 acquire_source;
                 u32 acquire_active;
                 u32 acquire_timeout;
                 u32 acquire_value;
-                u32 semaphore_off_val;
-                u32 semaphore_sequence;
-
-                struct {
-                    u32 smaphore_address_high;
-                    u32 smaphore_address_low;
-
-                    GPUVAddr SmaphoreAddress() const {
-                        return static_cast<GPUVAddr>(
-                            (static_cast<GPUVAddr>(smaphore_address_high) << 32) |
-                            smaphore_address_low);
-                    }
-                } smaphore_address;
             };
             std::array<u32, NUM_REGS> reg_array;
         };
@@ -222,15 +231,19 @@ private:
     static_assert(offsetof(GPU::Regs, field_name) == position * 4,                                 \
                   "Field " #field_name " has invalid position")
 
-ASSERT_REG_POSITION(acquire_mode, 0x0);
-ASSERT_REG_POSITION(acquire_source, 0x1);
-ASSERT_REG_POSITION(reference_count, 0x2);
-ASSERT_REG_POSITION(acquire_active, 0x3);
-ASSERT_REG_POSITION(acquire_timeout, 0x4);
-ASSERT_REG_POSITION(acquire_value, 0x5);
-ASSERT_REG_POSITION(semaphore_off_val, 0x6);
-ASSERT_REG_POSITION(semaphore_sequence, 0x7);
-ASSERT_REG_POSITION(smaphore_address, 0x8);
+ASSERT_REG_POSITION(smaphore_address, 0x4);
+ASSERT_REG_POSITION(semaphore_sequence, 0x6);
+ASSERT_REG_POSITION(semaphore_trigger, 0x7);
+ASSERT_REG_POSITION(reference_count, 0x14);
+ASSERT_REG_POSITION(semaphore_acquire, 0x1A);
+ASSERT_REG_POSITION(semaphore_release, 0x1B);
+
+ASSERT_REG_POSITION(acquire_mode, 0x100);
+ASSERT_REG_POSITION(acquire_source, 0x101);
+ASSERT_REG_POSITION(acquire_active, 0x102);
+ASSERT_REG_POSITION(acquire_timeout, 0x103);
+ASSERT_REG_POSITION(acquire_value, 0x104);
+
 #undef ASSERT_REG_POSITION
 
 } // namespace Tegra