Altering animation play state messes up my animation synching with audio

I am synching a div scrolling effect animation with an audio, the synching works fine when I set the element.style.animation property to 'none' when the audio is paused. The problem is when I try to pause the animation instead when the audio is paused. When I pause and play multiple times, the animation is not synched with the audio anymore (animation is too advanced).

//This is the audio element
const pupHymn = document.querySelector(".pup-hymn");
//This is the container div of the lyrics
const hymnLyrics = document.querySelector(".lyrics");

const init = () => {
    pupHymn.volume = 0.3;
    //Sync when playing manually 
    pupHymn.addEventListener('play', () => {
        audioTime = pupHymn.currentTime;
        playbackRate = pupHymn.playbackRate;
        hymnLyrics.style.animation = `scroll-effect ${160/playbackRate}s ${(12-audioTime)/playbackRate}s forwards running`;
    );

    pupHymn.addEventListener('pause', () => {
        hymnLyrics.style.animation = 'none'; 
    });

//This works fine.
pupHymn.addEventListener('pause', () => {
        hymnLyrics.style.animation = 'none'; });
}

But when I paused the animationPlayState instead, it won't synch properly when I pause and play multiple times.

pupHymn.addEventListener('pause', () => {
        hymnLyrics.style.animationPlayState = 'paused'; });

I already have tried logging the animation property of my div when the audio will be played, even though the delay is still a positive value, the animation started already.

The most reliable way here is probably through the Web Animation API, which allows you to manage the current time of your animation with relatively high precision. You can hook to some events on your <audio> element and map its .currentTime to the Animation's one (beware the former is in seconds while the latter is in milliseconds).

<!-- begin snippet: js hide: false console: true babel: false -->

<!-- language: lang-js -->

const audio = document.querySelector("audio");
const target = document.querySelector(".content");
audio.addEventListener("loadedmetadata", (evt) => {
  const { duration } = audio;
  const keys = new KeyframeEffect(
    target,
    [{ transform: "translate(0px, -1300px)" },],
    { duration: duration * 1000, fill: "forwards" }
  );
  const anim = new Animation(keys);
  const sync = () => anim.currentTime = audio.currentTime * 1000;
  audio.addEventListener("seeked", sync);
  audio.addEventListener("play", () => {
    sync();
    anim.play();
  });
  audio.addEventListener("pause", () => {
    anim.pause();
    sync();
  });
}, { once: true });
for (let i = 0; i < 46; i++) {
  const el = document.createElement("p");
  el.textContent = i;
  target.append(el);
}

<!-- language: lang-css -->

.container {
  width: 300px;
  height: 300px;
  border: 1px solid;
  overflow: hidden;
}
.content {
  height: 1600px;
  background-image: linear-gradient(to bottom, red, blue);
}
.content p {
  margin-top: 0;
}

<!-- language: lang-html -->

<audio controls src="https://upload.wikimedia.org/wikipedia/en/transcoded/d/dc/Strawberry_Fields_Forever_%28Beatles_song_-_sample%29.ogg/Strawberry_Fields_Forever_%28Beatles_song_-_sample%29.ogg.mp3"></audio>

<div class=container>
  <div class="content"></div>
</div>

<!-- end snippet -->