/*
 * (C) Copyright 1992, ..., 2014 the "DOSEMU-Development-Team".
 *
 * for details see file COPYING in the DOSEMU distribution
 */

/*
 * Purpose: memory mapping library.
 *
 * Authors: Stas Sergeev, Bart Oldeman.
 * Initially started by Hans Lermen, old copyrights below:
 */
/* file mapping.c
 *
 * generic mapping driver interface
 * (C) Copyright 2000, Hans Lermen, lermen@fgan.de
 *
 * NOTE: We handle _all_ memory mappings within the mapping drivers,
 *       mmap-type as well as IPC shm, except for X-mitshm (in X.c),
 *       which is a special case and doesn't involve DOS space atall.
 *
 *       If you ever need to do mapping within DOSEMU (except X-mitshm),
 *       _always_ use the interface supplied by the mapping drivers!
 *       ^^^^^^^^         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 */

#include "emu.h"
#include "xms.h"
#include "utilities.h"
#include "dos2linux.h"
#include "kvm.h"
#include "misc/smalloc.h"
#include "mapping.h"
#include "mpriv.h"
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <fcntl.h>
#ifdef __linux__
#include <linux/version.h>
#endif

enum { MEM_BASE, KVM_BASE, JIT_BASE,
#ifdef __i386__
       VM86_BASE,
#endif
       MAX_BASES };

struct aliasmap_s {
  unsigned char *ptr;
  int protect;
  unsigned int mapped:1;
};

struct hardware_ram {
  size_t base;
  dosaddr_t default_vbase;
  dosaddr_t vbase;
  size_t size;
  int type;
  struct aliasmap_s *aliasmap;
  struct hardware_ram *next;
};

static int init_done = 0;
static struct {
  unsigned char *base;
  size_t size;
  unsigned int nx:1;
} mem_bases[MAX_BASES];
unsigned char *_mem_base(void)
{
  assert(mem_bases[MEM_BASE].base != MAP_FAILED);
  return mem_bases[MEM_BASE].base;
}
unsigned char *_jit_base(void)
{
  assert(mem_bases[JIT_BASE].base != MAP_FAILED);
  return mem_bases[JIT_BASE].base;
}
uint8_t *lowmem_base;

static struct mappingdrivers *mappingdrv[] = {
  &mappingdriver_softmmu, /* first try softmmu (fullsim only!) */
#ifdef HAVE_MEMFD_CREATE
#if HAVE_DECL_MEMFD_CREATE
  &mappingdriver_mshm,  /* then try memfd mmap */
#endif
#endif
#ifdef HAVE_SHM_OPEN
  &mappingdriver_shm,   /* then shm_open which is usually broken */
#endif
  &mappingdriver_file, /* and then a temp file */
};

static struct mappingdrivers *mappingdriver;
static const struct mapping_hook *mapping_hook;

#define ALIAS_SIZE (LOWMEM_SIZE + HMASIZE)
static unsigned char *lowmem_aliasmap[ALIAS_SIZE/PAGE_SIZE];
struct hardware_ram;
static dosaddr_t do_get_hardware_ram(unsigned addr, uint32_t size,
	struct hardware_ram **r_hw);
static unsigned do_find_hardware_ram(dosaddr_t va, uint32_t size,
	struct hardware_ram **r_hw);
static void hwram_update_aliasmap(struct hardware_ram *hw, unsigned addr,
	int size, unsigned char *src);
static void hwram_map_aliasmap(struct hardware_ram *hw, unsigned addr,
	int size, int mapped);
static void hwram_mprotect_aliasmap(struct hardware_ram *hw, unsigned addr,
	int size, int protect);
static int hwram_is_mapped(struct hardware_ram *hw, unsigned addr, int size);
static int hwram_restore_mapping(struct hardware_ram *hw, unsigned addr,
	int size, dosaddr_t va);
static int hwram_prot_match(struct hardware_ram *hw, unsigned addr,
	int size, int prot);
static int is_kvm_map(int cap);
static void *do_huge_page(int flags, size_t mapsize, int protect);
#if HAVE_DECL_MADV_POPULATE_WRITE
static int madvise_mapping(dosaddr_t targ, size_t length, int flags);
#endif
static void update_aliasmap(dosaddr_t dosaddr, size_t mapsize,
			    unsigned char *unixaddr)
{
  unsigned dospage;
  int i;

  if (dosaddr >= ALIAS_SIZE)
    return;
  dospage = dosaddr >> PAGE_SHIFT;
  for (i = 0; i < mapsize >> PAGE_SHIFT; i++)
    lowmem_aliasmap[dospage + i] = unixaddr ? unixaddr + (i << PAGE_SHIFT) : NULL;
  invalidate_unprotected_page_cache(dosaddr, mapsize);
}

void *dosaddr_to_unixaddr(dosaddr_t addr)
{
  unsigned addr2;
  int off;
  struct hardware_ram *hw;

  if (addr < ALIAS_SIZE)
    return lowmem_aliasmap[addr >> PAGE_SHIFT] + (addr & (PAGE_SIZE - 1));
  addr2 = do_find_hardware_ram(addr, 1, &hw);
  /* since 517a4c61 lowmem_base follows VA, not PA */
  if (addr2 == (unsigned)-1)
    return LOWMEM(addr);
  off = addr - hw->vbase;
  if (hw->aliasmap[off >> PAGE_SHIFT].ptr)
    return hw->aliasmap[off >> PAGE_SHIFT].ptr + (off & (PAGE_SIZE - 1));
  return LOWMEM(addr);
}

void *physaddr_to_unixaddr(unsigned int addr)
{
  void *hwr = get_hardware_uaddr(addr);
  if (hwr != MAP_FAILED)
    return hwr;
  if (addr < ALIAS_SIZE)
    return dosaddr_to_unixaddr(addr);
  return MAP_FAILED;
}

dosaddr_t physaddr_to_dosaddr(unsigned int addr, int len)
{
  dosaddr_t ret = get_hardware_ram(addr, len);
  if (ret != (dosaddr_t)-1)
    return ret;
  if (addr + len <= ALIAS_SIZE)
    return addr;
  return -1;
}

static unsigned char *MEM_BASE32x(dosaddr_t a, int base)
{
  if (mem_bases[base].base == MAP_FAILED || a >= mem_bases[base].size)
    return MAP_FAILED;
  return &mem_bases[base].base[a];
}

static int is_kvm_map(int cap)
{
  if (config.cpu_vm != CPUVM_KVM && config.cpu_vm_dpmi != CPUVM_KVM)
    return 0;
  if (config.cpu_vm_dpmi == CPUVM_KVM)
    return 1;
  if (cap & MAPPING_INIT_LOWRAM)
    return 1;
  /* v86 kvm, dpmi native */
  return (!(cap & MAPPING_DPMI));
}

void *alias_mapping_ux(int cap, size_t mapsize, int protect, void *source)
{
  void *target = (void *)-1;
  return mappingdriver->alias(cap, target, mapsize, protect, source);
}

int alias_mapping_high(int cap, dosaddr_t targ, size_t mapsize, int protect,
    void *source)
{
  void *addr;

  if (mem_bases[MEM_BASE].base == MAP_FAILED)
    return 0;

  Q__printf("MAPPING: alias, cap=%s, targ=%#x, size=%zx, protect=%x, source=%p\n",
	cap, targ, mapsize, protect, source);
  addr = mappingdriver->alias(cap, MEM_BASE32(targ), mapsize, protect, source);
  if (addr == MAP_FAILED)
    return -1;
  if (cap & MAPPING_INIT_LOWRAM) {
    void *addr1 = do_huge_page(0, mapsize, protect);
    void *addr2 = MAP_FAILED;

    if (addr1 != MAP_FAILED)
      addr2 = mappingdriver->alias(cap, addr1, mapsize, protect,
        source);
    if (addr2 == MAP_FAILED) {
      error("second alias failed\n");
      munmap(addr, mapsize);
      return -1;
    }
    mem_bases[JIT_BASE].base = addr2;
    mem_bases[JIT_BASE].size = mapsize;
  }
  Q__printf("MAPPING: %s alias created at %p\n", cap, addr);
  if (is_kvm_map(cap))
    mprotect_kvm(cap, targ, mapsize, protect);
  return 0;
}

#ifdef __linux__
static int kmem_mapped(dosaddr_t addr, int mapsize)
{
  struct hardware_ram *hw;
  unsigned addr2 = do_find_hardware_ram(addr, mapsize, &hw);
  if (addr2 == (unsigned)-1)
    return 0;
  return hw->type == 'v' || hw->type == 'h';
}
#endif

int alias_mapping(int cap, dosaddr_t targ, size_t mapsize, int protect, void *source)
{
  int i;

  assert(targ != (dosaddr_t)-1);
  Q__printf("MAPPING: alias, cap=%s, targ=%#x, size=%zx, protect=%x, source=%p\n",
	cap, targ, mapsize, protect, source);
#ifdef __linux__
  assert(kmem_mapped(targ, mapsize) == 0);
#endif

  for (i = 0; i < MAX_BASES; i++) {
    void *target, *addr;
    int prot;
    target = MEM_BASE32x(targ, i);
    if (target == MAP_FAILED)
      continue;
    /* protections on KVM_BASE go via page tables in the VM, not mprotect */
    prot = i == KVM_BASE ? PROT_RW : protect;
    if (mem_bases[i].nx)
      prot &= ~PROT_EXEC;
    addr = mappingdriver->alias(cap, target, mapsize, prot, source);
    if (addr == MAP_FAILED)
      return -1;
    Q__printf("MAPPING: %s alias created at %p\n", cap, addr);
  }
  update_aliasmap(targ, mapsize, source);
  if (is_kvm_map(cap))
    mprotect_kvm(cap, targ, mapsize, protect);

  return 0;
}

static int mapping_is_hole(void *start, size_t size)
{
  uintptr_t beg = (uintptr_t)start;
  uintptr_t end = beg + size;
  return (mapping_find_hole(beg, end, size) == start);
}

// KVM api documentation recommends that the lower 21 bits of guest_phys_addr and
// userspace_addr be identical, so align to a huge page boundary (2MB).
// This also helps debugging, since subtracting hex numbers is much easier that way
// for a human being.
static void *do_huge_page(int flags, size_t mapsize, int protect)
{
  size_t edge;
  unsigned char *addr = mmap(NULL, mapsize + HUGE_PAGE_SIZE, protect,
			     MAP_PRIVATE | flags | MAP_ANONYMOUS, -1, 0);
  if (addr == MAP_FAILED)
    return MAP_FAILED;

  /* align up to next 2MB */
  edge = (unsigned char *)HUGE_PAGE_ALIGN((uintptr_t)addr + PAGE_SIZE) - addr;

  /* create guard page to trap bad things */
  mprotect(&addr[edge - PAGE_SIZE], PAGE_SIZE, PROT_NONE);

  /* trim front */
  if (edge > PAGE_SIZE)
    munmap(addr, edge - PAGE_SIZE);

  addr += edge;

  /* trim back */
  edge = HUGE_PAGE_SIZE - edge;
  if (edge > 0)
    munmap(&addr[mapsize], edge);

  return addr;
}

void *mmap_mapping_huge_page_aligned(int cap, size_t mapsize, int protect)
{
  int flags = (cap & MAPPING_INIT_LOWRAM) ? _MAP_32BIT : 0;
  void *addr = do_huge_page(flags, mapsize, protect);
  if (addr == MAP_FAILED)
    return addr;
  if (cap & MAPPING_INIT_LOWRAM) {
    mem_bases[MEM_BASE].base = addr;
    mem_bases[MEM_BASE].size = mapsize;
    if (config.cpu_vm_dpmi == CPUVM_NATIVE)
      mem_bases[MEM_BASE].nx = config.dpmi_remote;
    if (is_kvm_map(cap)) {
      mapsize = LOWMEM_SIZE + HMASIZE;
      protect = PROT_RWX;
      void *kvm_base = do_huge_page(0, mapsize, protect);
      if (kvm_base == MAP_FAILED)
	return kvm_base;
      mem_bases[KVM_BASE].base = kvm_base;
      mem_bases[KVM_BASE].size = mapsize;
    }
#ifdef __i386__
    if (config.cpu_vm == CPUVM_VM86) {
      mem_bases[VM86_BASE].base = 0;
      mem_bases[VM86_BASE].size = ALIAS_SIZE;
      mem_bases[VM86_BASE].nx = 0;
    }
#endif
  }
  return addr;
}

void *mmap_mapping(int cap, void *target, size_t mapsize, int protect)
{
  void *addr;
  int flags;

  assert(target != (void *)-1);

  /* not removing MAP_FIXED when mapping to 0 */
  if ((cap & MAPPING_NULL) && !mapping_is_hole(target, mapsize))
    return MAP_FAILED;

  flags = (cap & MAPPING_NOOVERLAP) ?
#ifdef MAP_FIXED_NOREPLACE
    MAP_FIXED_NOREPLACE
#else
    0
#endif
    : MAP_FIXED;

  addr = mmap(target, mapsize, protect,
	      MAP_PRIVATE | flags | MAP_ANONYMOUS, -1, 0);

  if (addr != MAP_FAILED && addr != target) {
#ifdef MAP_FIXED_NOREPLACE
#ifdef __linux__
    /* under valgrind this flag doesn't work */
    if (kernel_version_code >= KERNEL_VERSION(4, 17, 0))
      error_once0("MAP_FIXED_NOREPLACE doesn't work\n");
#endif
#endif
    munmap(addr, mapsize);
    return MAP_FAILED;
  }
  return addr;
}

int munmap_mapping(int cap, dosaddr_t targ, size_t mapsize)
{
  int i, ret = -1;

  for (i = 0; i < MAX_BASES; i++) {
    void *addr = MEM_BASE32x(targ, i);
    if (addr == MAP_FAILED)
      continue;
    ret = munmap(addr, mapsize);
    if (ret) {
      error("munmap() failed: %s\n", strerror(errno));
      return ret;
    }
  }
  return ret;
}

int munmap_mapping_pa(int cap, unsigned int addr, size_t mapsize)
{
  struct hardware_ram *hw;
  dosaddr_t va = do_get_hardware_ram(addr, mapsize, &hw);
  if (va == (dosaddr_t)-1)
    return -1;
  assert(addr >= GRAPH_BASE);
  if (!hwram_is_mapped(hw, addr, mapsize))
    return -1;
#ifdef __linux__
  if (kmem_mapped(addr, mapsize)) {
    error("not unmapping kmem from %x (size %zx)\n", addr, mapsize);
    return -1;
  }
#endif
  restore_mapping(cap, va, mapsize);
  hwram_map_aliasmap(hw, addr, mapsize, 0);
  return 0;
}

/* Restore mapping previously broken by direct mmap() call. */
int restore_mapping(int cap, dosaddr_t targ, size_t mapsize)
{
  int ret;
  assert((cap & (MAPPING_DPMI | MAPPING_VGAEMU | MAPPING_INIT_LOWRAM)) && (targ != (dosaddr_t)-1));
  ret = alias_mapping(cap, targ, mapsize, PROT_READ | PROT_WRITE,
      LOWMEM(targ));
  return ret;
}

int restore_mapping_pa(unsigned int addr, size_t mapsize)
{
  int err;
  struct hardware_ram *hw;
  dosaddr_t va = do_get_hardware_ram(addr, mapsize, &hw);
  if (va == (dosaddr_t)-1)
    return -1;
  assert(addr >= GRAPH_BASE);
  if (hwram_is_mapped(hw, addr, mapsize))
    return 0;
  err = hwram_restore_mapping(hw, addr, mapsize, va);
  if (err)
    return err;
  hwram_map_aliasmap(hw, addr, mapsize, 1);
  return 0;
}

static int do_mprot(dosaddr_t targ, size_t mapsize, int protect)
{
  int i, ret;

  for (i = 0; i < MAX_BASES; i++) {
    void *addr = MEM_BASE32x(targ, i);
    /* protections on KVM_BASE go via page tables in the VM, not mprotect */
    if (addr == MAP_FAILED || i == KVM_BASE)
      continue;
    if (mem_bases[i].nx)
      protect &= ~PROT_EXEC;
    ret = mprotect(addr, mapsize, protect);
    if (ret) {
      error("mprotect() failed: %s\n", strerror(errno));
      return ret;
    }
  }
  return 0;
}

int mprotect_mapping(int cap, dosaddr_t targ, size_t mapsize, int protect)
{
  int ret = -1;
  void *addr;

  Q__printf("MAPPING: mprotect, cap=%s, targ=%x, size=%zx, protect=%x\n",
	cap, targ, mapsize, protect);
  invalidate_unprotected_page_cache(targ, mapsize);
  if (cap & MAPPING_CPUEMU) {
    if (mem_bases[JIT_BASE].base == MAP_FAILED)
      return 0;
    /* for cpuemu only mprotect JIT_BASE */
    ret = mprotect(MEM_BASE32x(targ, JIT_BASE), mapsize, protect);
    if (ret)
      error("mprotect() failed: %s\n", strerror(errno));
    return ret;
  }
  ret = do_mprot(targ, mapsize, protect);
  if (ret)
    return ret;
  if (is_kvm_map(cap))
    mprotect_kvm(cap, targ, mapsize, protect);
  if (mem_bases[MEM_BASE].base == MAP_FAILED)
    return ret;
  addr = MEM_BASE32(targ);
  if ((unsigned char *)addr < mem_bases[MEM_BASE].base ||
      (unsigned char *)addr + mapsize > mem_bases[MEM_BASE].base +
      mem_bases[MEM_BASE].size)
    return ret;
  if (mapping_hook)
    ret = mapping_hook->mprotect(addr, mapsize, protect);
  return ret;
}

int mprotect_mapping_pa(unsigned int addr, size_t mapsize, int protect)
{
  struct hardware_ram *hw;
  int err;
  dosaddr_t va = do_get_hardware_ram(addr, mapsize, &hw);
  if (va == (dosaddr_t)-1)
    return -1;
  assert(addr >= GRAPH_BASE);
  if (hwram_prot_match(hw, addr, mapsize, protect))
    return 0;
  /* if not mapped, then don't touch */
  if (hwram_is_mapped(hw, addr, mapsize)) {
    err = mprotect_mapping(MAPPING_LOWMEM, va, mapsize, protect);
    if (err)
      return err;
  }
  hwram_mprotect_aliasmap(hw, addr, mapsize, protect);
  return 0;
}

/*
 * This gets called on DOSEMU startup to determine the kind of mapping
 * and setup the appropriate function pointers
 */
void mapping_init(void)
{
  int i;
  int numdrivers = sizeof(mappingdrv) / sizeof(mappingdrv[0]);

  assert(!init_done);
  init_done++;

  if (config.mappingdriver && strcmp(config.mappingdriver, "auto")) {
    /* first try the mapping driver the user wants */
    for (i=0; i < numdrivers; i++) {
      if (!strcmp(mappingdrv[i]->key, config.mappingdriver))
        break;
    }
    if (i >= numdrivers) {
      error("Wrong mapping driver specified: %s\n", config.mappingdriver);
      exit(2);
      return;
    }
    if (!mappingdrv[i]->open(MAPPING_PROBE)) {
      error("Failed to initialize mapping %s\n", config.mappingdriver);
      leavedos(2);
      return;
    }
    mappingdriver = mappingdrv[i];
  } else {
    for (i = 0; i < numdrivers; i++) {
      if (mappingdrv[i] && (*mappingdrv[i]->open)(MAPPING_PROBE)) {
        mappingdriver = mappingdrv[i];
        Q_printf("MAPPING: using the %s driver\n", mappingdriver->name);
        break;
      }
    }
    if (i >= numdrivers) {
      error("MAPPING: cannot allocate an appropriate mapping driver\n");
      leavedos(2);
      return;
    }
  }

  for (i = 0; i < MAX_BASES; i++) {
    mem_bases[i].base = MAP_FAILED;
    mem_bases[i].size = 0;
    mem_bases[i].nx = 1;
  }
}

/* this gets called on DOSEMU termination cleanup all mapping stuff */
void mapping_close(void)
{
  if (init_done && mappingdriver->close) close_mapping(MAPPING_ALL);
}

static char dbuf[256];
char *decode_mapping_cap(int cap)
{
  char *p = dbuf;
  p[0] = 0;
  if (!cap) return dbuf;
  if ((cap & MAPPING_ALL) == MAPPING_ALL) {
    p += sprintf(p, " ALL");
  }
  else {
    if (cap & MAPPING_OTHER) p += sprintf(p, " OTHER");
    if (cap & MAPPING_EMS) p += sprintf(p, " EMS");
    if (cap & MAPPING_DPMI) p += sprintf(p, " DPMI");
    if (cap & MAPPING_VGAEMU) p += sprintf(p, " VGAEMU");
    if (cap & MAPPING_VIDEO) p += sprintf(p, " VIDEO");
    if (cap & MAPPING_VC) p += sprintf(p, " VC");
    if (cap & MAPPING_SHM) p += sprintf(p, " SHM");
    if (cap & MAPPING_HMA) p += sprintf(p, " HMA");
    if (cap & MAPPING_SHARED) p += sprintf(p, " SHARED");
    if (cap & MAPPING_INIT_HWRAM) p += sprintf(p, " INIT_HWRAM");
    if (cap & MAPPING_INIT_LOWRAM) p += sprintf(p, " INIT_LOWRAM");
    if (cap & MAPPING_EXTMEM) p += sprintf(p, " EXTMEM");
    if (cap & MAPPING_KVM) p += sprintf(p, " KVM");
  }
  if (cap & MAPPING_KMEM) p += sprintf(p, " KMEM");
  if (cap & MAPPING_LOWMEM) p += sprintf(p, " LOWMEM");
  if (cap & MAPPING_SCRATCH) p += sprintf(p, " SCRATCH");
  if (cap & MAPPING_SINGLE) p += sprintf(p, " SINGLE");
  if (cap & MAPPING_NULL) p += sprintf(p, " NULL");
  return dbuf;
}

int open_mapping(int cap)
{
  return mappingdriver->open(cap);
}

void close_mapping(int cap)
{
  if (mappingdriver->close) mappingdriver->close(cap);
}

static void *do_alloc_mapping(int cap, size_t mapsize, void *addr)
{
  addr = mappingdriver->alloc(cap, mapsize, addr);
  if (addr == MAP_FAILED) {
    error("failed to alloc %zx\n", mapsize);
    exit(2);
    return NULL;
  }
  if (cap & MAPPING_LOWMEM) {
    Q__printf("MAPPING: LOWRAM_INIT, cap=%s, base=%p\n", cap, addr);
    lowmem_base = addr;
  }
  Q__printf("MAPPING: %s allocated at %p\n", cap, addr);
  return addr;
}

void *alloc_mapping(int cap, size_t mapsize)
{
  Q__printf("MAPPING: alloc, cap=%s size=%#zx\n", cap, mapsize);
  return do_alloc_mapping(cap, mapsize, (void *)-1);
}

void *alloc_mapping_huge_page_aligned(int cap, size_t mapsize)
{
  void *addr;
  Q__printf("MAPPING: alloc_huge_page_aligned, cap=%s size=%#zx\n", cap, mapsize);
  addr = do_huge_page(0, mapsize, PROT_NONE);
  return addr == MAP_FAILED ? MAP_FAILED : do_alloc_mapping(cap, mapsize, addr);
}

void free_mapping(int cap, void *addr, size_t mapsize)
{
  if (cap & MAPPING_KMEM) {
    return;
  }
  mprotect(addr, mapsize, PROT_READ | PROT_WRITE);
  mappingdriver->free(cap, addr, mapsize);
}

void *realloc_mapping(int cap, void *addr, size_t oldsize, size_t newsize)
{
  if (!addr) {
    if (oldsize)  // no-no, realloc of the lowmem is not good too
      dosemu_error("realloc_mapping() called with addr=NULL, oldsize=%#zx\n", oldsize);
    Q_printf("MAPPING: realloc from NULL changed to malloc\n");
    return alloc_mapping(cap, newsize);
  }
  if (!oldsize)
    dosemu_error("realloc_mapping() addr=%p, oldsize=0\n", addr);

  return mappingdriver->resize(cap, addr, oldsize, newsize);
}

static void init_aliasmap(struct aliasmap_s *map, int size)
{
  int i;

  for (i = 0; i < PAGE_ALIGN(size) >> PAGE_SHIFT; i++) {
    map[i].ptr = NULL;
    map[i].protect = PROT_READ | PROT_WRITE;
    map[i].mapped = 1;
  }
}

static void populate_aliasmap(struct aliasmap_s *map, unsigned char *addr,
	int size)
{
  int i;

  for (i = 0; i < PAGE_ALIGN(size) >> PAGE_SHIFT; i++)
    map[i].ptr = addr ? addr + (i << PAGE_SHIFT) : NULL;
}

static void map_aliasmap(struct aliasmap_s *map, int size, int mapped)
{
  int i;

  for (i = 0; i < PAGE_ALIGN(size) >> PAGE_SHIFT; i++)
    map[i].mapped = mapped;
}

static void protect_aliasmap(struct aliasmap_s *map, int size, int protect)
{
  int i;

  for (i = 0; i < PAGE_ALIGN(size) >> PAGE_SHIFT; i++)
    map[i].protect = protect;
}

static int is_mapped_aliasmap(struct aliasmap_s *map, int size)
{
  int i;

  for (i = 0; i < PAGE_ALIGN(size) >> PAGE_SHIFT; i++) {
    if (!map[i].mapped)
      return 0;
  }
  return 1;
}

static void ensure_unmapped_aliasmap(struct aliasmap_s *map, int size)
{
  int i;

  for (i = 0; i < PAGE_ALIGN(size) >> PAGE_SHIFT; i++)
    assert(!map[i].mapped);
}

static int restore_mapping_aliasmap(struct aliasmap_s *map, int size,
	dosaddr_t va)
{
  int i;

  for (i = 0; i < PAGE_ALIGN(size) >> PAGE_SHIFT; i++) {
    struct aliasmap_s *am = &map[i];
    dosaddr_t vad = va + (i << PAGE_SHIFT);
    int err;

    /* am->ptr==NULL means unaliasable mapping */
    assert(am->ptr);
    err = alias_mapping(MAPPING_LOWMEM, vad, PAGE_SIZE, am->protect, am->ptr);
    if (err)
      return err;
  }
  return 0;
}

static int prot_match_aliasmap(struct aliasmap_s *map, int size, int prot)
{
  int i;

  for (i = 0; i < PAGE_ALIGN(size) >> PAGE_SHIFT; i++) {
    if (map[i].protect != prot)
      return 0;
  }
  return 1;
}

static struct aliasmap_s *alloc_aliasmap(int size)
{
  struct aliasmap_s *ret = malloc((PAGE_ALIGN(size) >> PAGE_SHIFT) * sizeof(*ret));
  init_aliasmap(ret, size);
  return ret;
}

static struct hardware_ram *hardware_ram;

#if __linux__
static void *mmap_mapping_kmem(int cap, struct hardware_ram *hw)
{
    void *addr, *addr2;
    size_t mapsize = hw->size;
    off_t source = hw->base;
    dosaddr_t targ = hw->default_vbase;
    int i;

    if (source == -1) {
      error("KMEM mapping without source\n");
      leavedos(64);
    }

    if (!(cap & MAPPING_LOWMEM)) {
      if (!config.dpmi)
	return 0;
      targ = smalloc_aligned_topdown(&main_pool, (dosaddr_t)-1, PAGE_SIZE, mapsize);
      if (targ == (dosaddr_t)-1) {
	error("OOM for mmap_mapping_kmem, %s\n", strerror(errno));
	return MAP_FAILED;
      }
    }
    assert(targ != (dosaddr_t)-1);

    Q__printf("MAPPING: map kmem, cap=%s, target=%x, size=%zx, source=%#jx\n",
	      cap, targ, mapsize, (intmax_t)source);

    open_kmem();
    for (i = 0; i < MAX_BASES; i++) {
      void *dst = MEM_BASE32x(targ, i);
      if (dst == MAP_FAILED)
	continue;
      addr = mmap(dst, mapsize, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED,
		  mem_fd, source);
      if (addr == MAP_FAILED) {
        close_kmem();
        error("mmap error in mmap_mapping_kmem %s\n", strerror (errno));
        return MAP_FAILED;
      }
    }
    addr2 = mmap(0, mapsize, PROT_READ | PROT_WRITE, MAP_SHARED,
		mem_fd, source);
    close_kmem();
    if (addr2 == MAP_FAILED) {
      error("mmap error in mmap_mapping_kmem %s\n", strerror (errno));
      return MAP_FAILED;
    }

    Q_printf("MAPPING: backing region allocated at %p\n", addr2);
    if (is_kvm_map(cap))
      mprotect_kvm(cap, targ, mapsize, PROT_READ | PROT_WRITE);
    update_aliasmap(targ, mapsize, addr2);

    hw->vbase = targ;
    g_printf("mapped hardware ram at 0x%08zx .. 0x%08zx at %#x\n",
	     hw->base, hw->base+hw->size-1, hw->vbase);
    return addr2;
}

/*
 * DANG_BEGIN_FUNCTION init_hardware_ram
 *
 * description:
 *  Initialize the hardware direct-mapped pages
 *
 * DANG_END_FUNCTION
 */
void init_hardware_ram(void)
{
  struct hardware_ram *hw;

  for (hw = hardware_ram; hw != NULL; hw = hw->next) {
    unsigned char *uaddr;
    int cap = MAPPING_KMEM;
    if (hw->vbase != (dosaddr_t)-1)  /* virtual hardware ram mapped later */
      continue;
    if (hw->default_vbase != (dosaddr_t)-1)
      cap |= MAPPING_LOWMEM;
    uaddr = mmap_mapping_kmem(cap, hw);
    if (uaddr == MAP_FAILED) {
      error("failed to map KMEM at %zx, size %zx: %s\n", hw->base, hw->size,
          strerror(errno));
      continue;
    }
    populate_aliasmap(hw->aliasmap, uaddr, hw->size);
  }
}

/* why count ??? */
/* Because you do not want to open it more than once! */
static u_char kmem_open_count = 0;

void
open_kmem (void)
{
  PRIV_SAVE_AREA
  /* as I understad it, /dev/kmem is the kernel's view of memory,
     * and /dev/mem is the identity-mapped (i.e. physical addressed)
     * memory. Currently under Linux, both are the same.
     */

  kmem_open_count++;

  if (mem_fd != -1)
    return;
  enter_priv_on();
  mem_fd = open("/dev/mem", O_RDWR | O_CLOEXEC);
  leave_priv_setting();
  if (mem_fd < 0)
    {
      error("can't open /dev/mem: errno=%d, %s \n",
	     errno, strerror (errno));
      leavedos (0);
      return;
    }
  g_printf ("Kmem opened successfully\n");
}

void
close_kmem (void)
{

  if (kmem_open_count)
    {
      kmem_open_count--;
      if (kmem_open_count)
	return;
      close (mem_fd);
      mem_fd = -1;
      v_printf ("Kmem closed successfully\n");
    }
}
#endif

static int do_register_hwram(int type, unsigned base, unsigned size,
	dosaddr_t va)
{
  struct hardware_ram *hw;

  c_printf("Registering HWRAM, type=%c base=%#x size=%#x\n", type, base, size);
  hw = malloc(sizeof(*hw));
  hw->base = base;
  if (base < LOWMEM_SIZE)
    hw->default_vbase = hw->base;
  else
    hw->default_vbase = -1;
  hw->vbase = va;
  hw->size = size;
  hw->type = type;
  hw->aliasmap = alloc_aliasmap(size);
  hw->next = hardware_ram;
  hardware_ram = hw;
  if (base >= LOWMEM_SIZE || type == 'h')
    memcheck_reserve(type, base, size);
  return 1;
}

int register_hardware_ram(int type, unsigned base, unsigned int size)
{
  if (!can_do_root_stuff) {
    dosemu_error("can't use hardware ram in low feature (non-suid root) DOSEMU\n");
    return 0;
  }
  return do_register_hwram(type, base, size, -1);
}

static void register_hardware_ram_virtual2(int type, unsigned base,
	unsigned int size, void *uaddr, dosaddr_t va)
{
  do_register_hwram(type, base, size, va);
  if (config.cpu_vm_dpmi == CPUVM_KVM ||
      (config.cpu_vm == CPUVM_KVM && base + size <= LOWMEM_SIZE + HMASIZE)) {
    int prot = KVM_PROT_RWX;
    int cap = MAPPING_INIT_LOWRAM;
    if (type == 'L')
      cap |= MAPPING_LOWMEM;
    if (base + size > ALIAS_SIZE)
      cap |= MAPPING_EXTMEM;
    mmap_kvm(cap, base, size, uaddr, va, prot);
  }
}

void register_hardware_ram_virtual(int type, unsigned base, unsigned int size,
	dosaddr_t va)
{
  void *kuaddr = NULL;

  if (config.cpu_vm_dpmi == CPUVM_KVM || config.cpu_vm == CPUVM_KVM)
    kuaddr = base < mem_bases[KVM_BASE].size ?
	MEM_BASE32x(base, KVM_BASE) : MEM_BASE32(va);
  register_hardware_ram_virtual2(type, base, size, kuaddr, va);
}

int unregister_hardware_ram_virtual(unsigned base)
{
  struct hardware_ram *hw, *phw;

  for (phw = NULL, hw = hardware_ram; hw != NULL; phw = hw, hw = hw->next) {
    if (hw->base == base) {
      if (phw)
        phw->next = hw->next;
      else
        hardware_ram = hw->next;
      if (hw->base + hw->size > ALIAS_SIZE)
        free(hw->aliasmap);
      free(hw);
      return 0;
    }
  }
  return -1;
}

/* given physical address addr, gives the corresponding vbase or -1 */
static dosaddr_t do_get_hardware_ram(unsigned addr, uint32_t size,
	struct hardware_ram **r_hw)
{
  struct hardware_ram *hw;

  for (hw = hardware_ram; hw != NULL; hw = hw->next) {
    if (hw->vbase != -1 &&
	hw->base <= addr && addr + size <= hw->base + hw->size) {
	if (r_hw)
	  *r_hw = hw;
      return hw->vbase + addr - hw->base;
    }
  }
  return -1;
}

static unsigned do_find_hardware_ram(dosaddr_t va, uint32_t size,
	struct hardware_ram **r_hw)
{
  struct hardware_ram *hw;

  for (hw = hardware_ram; hw != NULL; hw = hw->next) {
    if (hw->vbase == -1)
      continue;
    if (hw->vbase <= va && va + size <= hw->vbase + hw->size) {
      unsigned addr = hw->base + va - hw->vbase;
      if (!hwram_is_mapped(hw, addr & _PAGE_MASK, PAGE_ALIGN(size))) {
        /* make sure its not kmem somehow unmapped */
        assert(hw->type != 'v' && hw->type != 'h');
        return -1;
      }
      if (r_hw)
        *r_hw = hw;
      return addr;
    }
  }
  return -1;
}

dosaddr_t get_hardware_ram(unsigned addr, uint32_t size)
{
  return do_get_hardware_ram(addr, size, NULL);
}

static void hwram_update_aliasmap(struct hardware_ram *hw, unsigned addr,
	int size, unsigned char *src)
{
  int off = addr - hw->base;
  assert(!(off & (PAGE_SIZE - 1))); // page-aligned
  // lowmem needs permanent aliasing
  assert(!(src == NULL && (hw->base + hw->size <= ALIAS_SIZE)));
  populate_aliasmap(&hw->aliasmap[off >> PAGE_SHIFT], src, size);
}

static void hwram_map_aliasmap(struct hardware_ram *hw, unsigned addr,
	int size, int mapped)
{
  int off = addr - hw->base;
  assert(!(off & (PAGE_SIZE - 1))); // page-aligned
  map_aliasmap(&hw->aliasmap[off >> PAGE_SHIFT], size, mapped);
}

static void hwram_mprotect_aliasmap(struct hardware_ram *hw, unsigned addr,
	int size, int protect)
{
  int off = addr - hw->base;
  assert(!(off & (PAGE_SIZE - 1))); // page-aligned
  protect_aliasmap(&hw->aliasmap[off >> PAGE_SHIFT], size, protect);
}

static int hwram_is_mapped(struct hardware_ram *hw, unsigned addr, int size)
{
  int ret;
  int off = addr - hw->base;
  assert(!(off & (PAGE_SIZE - 1))); // page-aligned
  ret = is_mapped_aliasmap(&hw->aliasmap[off >> PAGE_SHIFT], size);
  if (!ret)
    ensure_unmapped_aliasmap(&hw->aliasmap[off >> PAGE_SHIFT], size);
  return ret;
}

static int hwram_restore_mapping(struct hardware_ram *hw, unsigned addr,
	int size, dosaddr_t va)
{
  int off = addr - hw->base;
  assert(!(off & (PAGE_SIZE - 1))); // page-aligned
  return restore_mapping_aliasmap(&hw->aliasmap[off >> PAGE_SHIFT], size, va);
}

static int hwram_prot_match(struct hardware_ram *hw, unsigned addr,
	int size, int prot)
{
  int off = addr - hw->base;
  assert(!(off & (PAGE_SIZE - 1))); // page-aligned
  return prot_match_aliasmap(&hw->aliasmap[off >> PAGE_SHIFT], size, prot);
}

void *get_hardware_uaddr(unsigned addr)
{
  struct hardware_ram *hw;

  for (hw = hardware_ram; hw != NULL; hw = hw->next) {
    if (hw->vbase != -1 &&
	hw->base <= addr && addr < hw->base + hw->size) {
      int off = addr - hw->base;
      if (hw->aliasmap[off >> PAGE_SHIFT].ptr)
        return hw->aliasmap[off >> PAGE_SHIFT].ptr + (off & (PAGE_SIZE - 1));
      return LOWMEM(addr);
    }
  }
  return MAP_FAILED;
}

void list_hardware_ram(void (*print)(const char *, ...))
{
  struct hardware_ram *hw;

  (*print)("hardware_ram: %s\n", hardware_ram ? "" : "no");
  if (!hardware_ram) return;
  (*print)("hardware_pages:\n");
  for (hw = hardware_ram; hw != NULL; hw = hw->next)
    (*print)("%08x-%08x\n", hw->base, hw->base + hw->size - 1);
}

void *mapping_find_hole(unsigned long start, unsigned long stop,
	unsigned long size)
{
    FILE *fp;
    unsigned long beg, end, pend;
    int ret;

    /* find out whether the address request is available */
    fp = dosemu_proc_self_maps;
    if (!fp)
	return MAP_FAILED;
    fseek(fp, 0, SEEK_SET);
    pend = start;
    while ((ret = fscanf(fp, "%lx-%lx%*[^\n]", &beg, &end)) == 2) {
	if (beg <= start) {
	    if (end > pend)
		pend = end;
	    continue;
	}
	if (beg - pend >= size)
	    break;
	if (end + size > stop) {
	    return MAP_FAILED;
	}
	pend = end;
    }
    if (ret != 2)
	return MAP_FAILED;
    return (void *)pend;
}

int mcommit_mapping(dosaddr_t targ, size_t size)
{
  int err = 0;
#if HAVE_DECL_MADV_POPULATE_WRITE
  err = madvise_mapping(targ, size, MADV_POPULATE_WRITE);
  if (err)
    perror("madvise()");
#endif
  return err;
}

int mcommit(void *addr, size_t size)
{
  int err = 0;
#if HAVE_DECL_MADV_POPULATE_WRITE
  err = madvise(addr, size, MADV_POPULATE_WRITE);
  if (err)
    perror("madvise()");
#endif
  return err;
}

int alias_mapping_vapa(int cap, dosaddr_t targ, size_t mapsize, int protect,
    unsigned paddr)
{
  int err;
  struct hardware_ram *hw;
  dosaddr_t va = do_get_hardware_ram(paddr, mapsize, &hw);
  if (va == (dosaddr_t)-1)
    return -1;
  err = hwram_restore_mapping(hw, paddr, mapsize, targ);
  if (err)
    return err;
  return mprotect_mapping(cap, targ, mapsize, protect);
}

int alias_mapping_pa(int cap, unsigned addr, size_t mapsize, int protect,
       void *source)
{
  struct hardware_ram *hw;
  int err;
  dosaddr_t va = do_get_hardware_ram(addr, mapsize, &hw);
  if (va == (dosaddr_t)-1)
    return -1;
  assert(addr >= GRAPH_BASE);
  /* if not mapped, then don't touch */
  if (hwram_is_mapped(hw, addr, mapsize)) {
    err = alias_mapping(cap, va, mapsize, protect, source);
    if (err)
      return err;
  }
  hwram_update_aliasmap(hw, addr, mapsize, source);
  hwram_mprotect_aliasmap(hw, addr, mapsize, protect);
  if ((cap & MAPPING_INIT_LOWRAM) && (config.cpu_vm_dpmi == CPUVM_KVM ||
      (config.cpu_vm == CPUVM_KVM && addr + mapsize <= LOWMEM_SIZE + HMASIZE))) {
    int prot = KVM_PROT_RWX;
    if (addr + mapsize > ALIAS_SIZE)
      cap |= MAPPING_EXTMEM;
    mmap_kvm(cap, addr, mapsize, source, va, prot);
  }
  return 0;
}

int unalias_mapping_pa(int cap, unsigned addr, size_t mapsize)
{
  struct hardware_ram *hw;
  dosaddr_t va = do_get_hardware_ram(addr, mapsize, &hw);
  if (va == (dosaddr_t)-1)
    return -1;
  assert(addr >= LOWMEM_SIZE + HMASIZE);
  restore_mapping(cap, va, mapsize);
  hwram_update_aliasmap(hw, addr, mapsize, NULL);
  return 0;
}

int mapping_is_mapped_pa(unsigned int addr, int mapsize)
{
  struct hardware_ram *hw;
  dosaddr_t va = do_get_hardware_ram(addr, mapsize, &hw);
  if (va == (dosaddr_t)-1)
    return 0;
  assert(addr >= GRAPH_BASE);
  return hwram_is_mapped(hw, addr & _PAGE_MASK, PAGE_ALIGN(mapsize));
}

void mapping_register_hook(const struct mapping_hook *hook)
{
  assert(!mapping_hook);
  mapping_hook = hook;
}

void *mmap_shm_hook(int cap, void *addr, size_t length, int prot, int flags,
                    int fd, off_t offset)
{
  int err = 0;
  void *ret = mmap(addr, length, prot, flags, fd, offset);
  if (ret != addr || (unsigned char *)addr < mem_bases[MEM_BASE].base ||
      (unsigned char *)addr + length > mem_bases[MEM_BASE].base +
      mem_bases[MEM_BASE].size)
    return ret;
  if (mapping_hook)
    err = mapping_hook->mmap(addr, length, prot, flags, fd, offset);
  if (err) {
    munmap(ret, length);
    ret = MAP_FAILED;
  }
  return ret;
}

#if HAVE_DECL_MADV_POPULATE_WRITE
static int madvise_mapping(dosaddr_t targ, size_t length, int flags)
{
  int i, err;
  void *addr;

  for (i = 0; i < MAX_BASES; i++) {
    addr = MEM_BASE32x(targ, i);
    if (addr == MAP_FAILED)
      continue;
    err = madvise(addr, length, flags);
    if (err)
      return err;
  }
  addr = MEM_BASE32(targ);
  if ((unsigned char *)addr < mem_bases[MEM_BASE].base ||
      (unsigned char *)addr + length > mem_bases[MEM_BASE].base +
      mem_bases[MEM_BASE].size)
    return err;
  if (mapping_hook)
    err = mapping_hook->madvise(addr, length, flags);
  return err;
}
#endif

void *mmap_shm_mapping(size_t length, int prot, int fd)
{
  return mappingdriver->attach(fd, length, prot);
}

void munmap_shm_mapping(void *addr)
{
  mappingdriver->detach(addr);
}

int mprotect_vga(int idx, dosaddr_t targ, size_t mapsize, int protect)
{
  int wp = !(protect & PROT_WRITE);
  int err;

  err = do_mprot(targ, mapsize, protect);
  if (err)
    return err;
  if (mapping_hook)
    err = mapping_hook->wp_vga(idx, MEM_BASE32(targ), mapsize, wp);
  return err;
}

int mprotect_vga_pa(int idx, unsigned int addr, size_t mapsize, int protect)
{
  struct hardware_ram *hw;
  dosaddr_t va = do_get_hardware_ram(addr, mapsize, &hw);
  if (va == (dosaddr_t)-1)
    return -1;
  assert(addr >= GRAPH_BASE);
  if (hwram_prot_match(hw, addr, mapsize, protect))
    return 0;
  /* if not mapped, then don't touch */
  if (hwram_is_mapped(hw, addr, mapsize)) {
    int err = mprotect_vga(idx, va, mapsize, protect);
    if (err)
      return err;
  }
  hwram_mprotect_aliasmap(hw, addr, mapsize, protect);
  return 0;
}

int mapping_is_mapped(void *addr)
{
  int i;

  for (i = 0; i < MAX_BASES; i++) {
    if ((unsigned char *)addr >= mem_bases[i].base &&
        (unsigned char *)addr < mem_bases[i].base + mem_bases[i].size)
      return 1;
  }
  return 0;
}
