Re: [PATCH v2 14/19] RFC: accel/tcg: Support split-rwx for darwin/iOS with vm_remap

[Joelle van Dyne]

Another change I made in alloc_code_gen_buffer_mirror_vmremap (in my
patch as well) is to remove VM_FLAGS_RANDOM_ADDR. This was causing a
rare out of memory error whenever the random address it chooses is too
high.

-j

On Sat, Oct 31, 2020 at 6:42 PM Joelle van Dyne <j@getutm.app> wrote:
>
> There's a compiler warning:
>
> warning: incompatible pointer to integer conversion assigning to
> 'mach_vm_address_t' (aka 'unsigned long long') from 'void *'
> [-Wint-conversion]
>     buf_rw = tcg_ctx->code_gen_buffer;
>
> I changed it to
>     buf_rw = (mach_vm_address_t)tcg_ctx->code_gen_buffer;
>
> Also, MAP_JIT doesn't work with the split mapping (it needs the same
> entitlements that allows for RWX mapping) so I made the following
> changes
>
> @@ -1088,15 +1094,11 @@ static bool alloc_code_gen_buffer(size_t size,
> int mirror, Error **errp)
>      return true;
>  }
>  #else
> -static bool alloc_code_gen_buffer_anon(size_t size, int prot, Error **errp)
> +static bool alloc_code_gen_buffer_anon(size_t size, int prot, int
> flags, Error **errp)
>  {
> -    int flags = MAP_PRIVATE | MAP_ANONYMOUS;
>      void *buf;
>
> -#ifdef CONFIG_DARWIN
> -    /* Applicable to both iOS and macOS (Apple Silicon). */
> -    flags |= MAP_JIT;
> -#endif
> +    flags |= MAP_PRIVATE | MAP_ANONYMOUS;
>
>      buf = mmap(NULL, size, prot, flags, -1, 0);
>      if (buf == MAP_FAILED) {
> @@ -1211,7 +1213,7 @@ static bool
> alloc_code_gen_buffer_mirror_vmremap(size_t size, Error **errp)
>      vm_prot_t cur_prot, max_prot;
>
>      /* Map the read-write portion via normal anon memory. */
> -    if (!alloc_code_gen_buffer_anon(size, PROT_READ | PROT_WRITE, errp)) {
> +    if (!alloc_code_gen_buffer_anon(size, PROT_READ | PROT_WRITE, 0, errp)) {
>          return false;
>      }
>
> @@ -1263,6 +1265,8 @@ static bool alloc_code_gen_buffer_mirror(size_t
> size, Error **errp)
>
>  static bool alloc_code_gen_buffer(size_t size, int mirror, Error **errp)
>  {
> +    int flags = 0;
> +
>      if (mirror) {
>          Error *local_err = NULL;
>          if (alloc_code_gen_buffer_mirror(size, &local_err)) {
> @@ -1283,8 +1287,11 @@ static bool alloc_code_gen_buffer(size_t size,
> int mirror, Error **errp)
>      /* The tcg interpreter does not need execute permission. */
>      prot = PROT_READ | PROT_WRITE;
>  #endif
> +#ifdef CONFIG_DARWIN
> +    flags |= MAP_JIT;
> +#endif
>
> -    return alloc_code_gen_buffer_anon(size, prot, errp);
> +    return alloc_code_gen_buffer_anon(size, prot, flags, errp);
>  }
>  #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
>
> With this in addition to the iOS host patches, I was able to run it on
> the iPad but am getting random crashes that I am continuing to debug.
>
> -j
>
> On Thu, Oct 29, 2020 at 5:49 PM Richard Henderson
> <richard.henderson@linaro.org> wrote:
> >
> > Cribbed from code posted by Joelle van Dyne <j@getutm.app>,
> > and rearranged to a cleaner structure.  Completely untested.
> >
> > Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
> > ---
> >  accel/tcg/translate-all.c | 68 ++++++++++++++++++++++++++++++++++++++-
> >  1 file changed, 67 insertions(+), 1 deletion(-)
> >
> > diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c
> > index 3e69ebd1d3..bf8263fdb4 100644
> > --- a/accel/tcg/translate-all.c
> > +++ b/accel/tcg/translate-all.c
> > @@ -1093,6 +1093,11 @@ static bool alloc_code_gen_buffer_anon(size_t size, 
> > int prot, Error **errp)
> >      int flags = MAP_PRIVATE | MAP_ANONYMOUS;
> >      void *buf;
> >
> > +#ifdef CONFIG_DARWIN
> > +    /* Applicable to both iOS and macOS (Apple Silicon). */
> > +    flags |= MAP_JIT;
> > +#endif
> > +
> >      buf = mmap(NULL, size, prot, flags, -1, 0);
> >      if (buf == MAP_FAILED) {
> >          error_setg_errno(errp, errno,
> > @@ -1182,13 +1187,74 @@ static bool 
> > alloc_code_gen_buffer_mirror_memfd(size_t size, Error **errp)
> >      qemu_madvise(buf_rx, size, QEMU_MADV_HUGEPAGE);
> >      return true;
> >  }
> > -#endif
> > +#endif /* CONFIG_LINUX */
> > +
> > +#ifdef CONFIG_DARWIN
> > +#include <mach/mach.h>
> > +
> > +extern kern_return_t mach_vm_remap(vm_map_t target_task,
> > +                                   mach_vm_address_t *target_address,
> > +                                   mach_vm_size_t size,
> > +                                   mach_vm_offset_t mask,
> > +                                   int flags,
> > +                                   vm_map_t src_task,
> > +                                   mach_vm_address_t src_address,
> > +                                   boolean_t copy,
> > +                                   vm_prot_t *cur_protection,
> > +                                   vm_prot_t *max_protection,
> > +                                   vm_inherit_t inheritance);
> > +
> > +static bool alloc_code_gen_buffer_mirror_vmremap(size_t size, Error **errp)
> > +{
> > +    kern_return_t ret;
> > +    mach_vm_address_t buf_rw, buf_rx;
> > +    vm_prot_t cur_prot, max_prot;
> > +
> > +    /* Map the read-write portion via normal anon memory. */
> > +    if (!alloc_code_gen_buffer_anon(size, PROT_READ | PROT_WRITE, errp)) {
> > +        return false;
> > +    }
> > +
> > +    buf_rw = tcg_ctx->code_gen_buffer;
> > +    buf_rx = 0;
> > +    ret = mach_vm_remap(mach_task_self(),
> > +                        &buf_rx,
> > +                        size,
> > +                        0,
> > +                        VM_FLAGS_ANYWHERE | VM_FLAGS_RANDOM_ADDR,
> > +                        mach_task_self(),
> > +                        buf_rw,
> > +                        false,
> > +                        &cur_prot,
> > +                        &max_prot,
> > +                        VM_INHERIT_NONE);
> > +    if (ret != KERN_SUCCESS) {
> > +        /* TODO: Convert "ret" to a human readable error message. */
> > +        error_setg(errp, "vm_remap for jit mirror failed");
> > +        munmap((void *)buf_rw, size);
> > +        return false;
> > +    }
> > +
> > +    if (mprotect((void *)buf_rx, size, PROT_READ | PROT_EXEC) != 0) {
> > +        error_setg_errno(errp, errno, "mprotect for jit mirror");
> > +        munmap((void *)buf_rx, size);
> > +        munmap((void *)buf_rw, size);
> > +        return false;
> > +    }
> > +
> > +    tcg_rx_mirror_diff = buf_rx - buf_rw;
> > +    return true;
> > +}
> > +#endif /* CONFIG_DARWIN */
> >
> >  static bool alloc_code_gen_buffer_mirror(size_t size, Error **errp)
> >  {
> >      if (TCG_TARGET_SUPPORT_MIRROR) {
> >  #ifdef CONFIG_LINUX
> >          return alloc_code_gen_buffer_mirror_memfd(size, errp);
> > +#endif
> > +#ifdef CONFIG_DARWIN
> > +        return alloc_code_gen_buffer_mirror_vmremap(size, errp);
> >  #endif
> >      }
> >      error_setg(errp, "jit split-rwx not supported");
> > --
> > 2.25.1
> >