Use scanline for relocator to allow multiple memory sources

This commit is contained in:
Vladimir 'phcoder' Serbinenko 2010-04-02 18:43:52 +02:00
parent 8b0800f66f
commit accbdc88a5

View file

@ -22,7 +22,6 @@
#include <grub/misc.h>
#include <grub/cache.h>
/* TODO: use more efficient data structures if necessary. */
/* FIXME: check memory map. */
/* FIXME: try to request memory from firmware. */
@ -35,14 +34,22 @@ struct grub_relocator
grub_size_t relocators_size;
};
struct grub_relocator_subchunk
{
enum {CHUNK_TYPE_IN_REGION, CHUNK_TYPE_REGION_START} type;
grub_addr_t host_start;
grub_addr_t start;
grub_size_t size;
};
struct grub_relocator_chunk
{
struct grub_relocator_chunk *next;
grub_addr_t src;
grub_addr_t target;
grub_size_t size;
enum {CHUNK_TYPE_IN_REGION, CHUNK_TYPE_REGION_START} type;
grub_addr_t host_start;
struct grub_relocator_subchunk *subchunks;
unsigned nsubchunks;
};
struct grub_relocator *
@ -63,210 +70,52 @@ grub_relocator_new (void)
return ret;
}
static grub_mm_header_t
get_best_header (struct grub_relocator *rel,
grub_addr_t start, grub_addr_t end, grub_addr_t align,
grub_size_t size,
grub_mm_region_t rb, grub_mm_header_t *prev,
grub_addr_t *best_addr, int from_low_priv, int collisioncheck)
struct event
{
grub_mm_header_t h, hp;
grub_mm_header_t hb = NULL, hbp = NULL;
enum {
IN_REG_START = 0,
IN_REG_END = 1,
REG_BEG_START = 2,
REG_BEG_END = REG_BEG_START | 1,
COLLISION_START = 4,
COLLISION_END = COLLISION_START | 1
} type;
grub_addr_t pos;
union
{
struct
{
grub_mm_region_t reg;
grub_mm_header_t hancestor;
grub_mm_region_t *regancestor;
grub_mm_header_t head;
};
};
};
auto void try_addr (grub_addr_t allowable_start, grub_addr_t allowable_end);
void try_addr (grub_addr_t allowable_start, grub_addr_t allowable_end)
{
if (from_low_priv)
{
grub_addr_t addr;
#define DIGITSORT_BITS 8
#define DIGITSORT_MASK ((1 << DIGITSORT_BITS) - 1)
#define BITS_IN_BYTE 8
addr = ALIGN_UP (allowable_start, align);
#define max(a, b) (((a) > (b)) ? (a) : (b))
#define min(a, b) (((a) < (b)) ? (a) : (b))
if (addr < start)
addr = ALIGN_UP (start, align);
if (collisioncheck)
while (1)
static inline int
is_start (int type)
{
struct grub_relocator_chunk *chunk;
for (chunk = rel->chunks; chunk; chunk = chunk->next)
if ((chunk->target <= addr
&& addr < chunk->target + chunk->size)
|| (chunk->target < addr + size
&& addr + size < chunk->target + chunk->size)
|| (addr <= chunk->target && chunk->target < addr + size)
|| (addr < chunk->target + chunk->size
&& chunk->target + chunk->size < addr + size))
{
grub_dprintf ("relocator",
"collision 0x%llx+0x%llx, 0x%llx+0x%llx\n",
(unsigned long long) chunk->target,
(unsigned long long) chunk->size,
(unsigned long long) addr,
(unsigned long long) size);
addr = ALIGN_UP (chunk->target + chunk->size, align);
break;
}
if (!chunk)
break;
return !(type & 1) && (type != COLLISION_START);
}
if (allowable_end <= addr + size)
return;
if (addr > end)
return;
if (hb == NULL || *best_addr > addr)
static void
allocate_regstart (grub_addr_t addr, grub_size_t size, grub_mm_region_t rb,
grub_mm_region_t *regancestor, grub_mm_header_t hancestor)
{
hb = h;
hbp = hp;
*best_addr = addr;
grub_dprintf ("relocator", "picked %p/%lx\n", hb,
(unsigned long) addr);
}
}
else
{
grub_addr_t addr;
addr = ALIGN_DOWN (allowable_end - size, align);
if (addr > end)
addr = ALIGN_DOWN (end, align);
if (collisioncheck)
while (1)
{
struct grub_relocator_chunk *chunk;
for (chunk = rel->chunks; chunk; chunk = chunk->next)
if ((chunk->target <= addr
&& addr < chunk->target + chunk->size)
|| (chunk->target < addr + size
&& addr + size < chunk->target + chunk->size)
|| (addr <= chunk->target && chunk->target < addr + size)
|| (addr < chunk->target + chunk->size
&& chunk->target + chunk->size < addr + size))
{
addr = ALIGN_DOWN (chunk->target - size, align);
break;
}
if (!chunk)
break;
}
if (allowable_start > addr)
return;
if (addr < start)
return;
if (hb == NULL || *best_addr < addr)
{
hb = h;
hbp = hp;
*best_addr = addr;
grub_dprintf ("relocator", "picked %p/%lx\n", hb,
(unsigned long) addr);
}
}
}
hp = rb->first;
h = hp->next;
grub_dprintf ("relocator", "alive\n");
do
{
grub_addr_t allowable_start, allowable_end;
allowable_start = (grub_addr_t) h;
allowable_end = (grub_addr_t) (h + h->size);
if (h->magic != GRUB_MM_FREE_MAGIC)
grub_fatal ("free magic is broken at %p: 0x%x", h, h->magic);
try_addr (allowable_start, allowable_end);
if ((grub_addr_t) h == (grub_addr_t) (rb + 1))
{
grub_dprintf ("relocator", "Trying region start 0x%llx\n",
(unsigned long long) (allowable_start
- sizeof (*rb) - rb->pre_size));
try_addr (allowable_start - sizeof (*rb) - rb->pre_size,
allowable_end - sizeof (*rb));
}
hp = h;
h = hp->next;
}
while (hp && hp != rb->first);
*prev = hbp;
return hb;
}
static int
malloc_in_range (struct grub_relocator *rel,
grub_addr_t start, grub_addr_t end, grub_addr_t align,
grub_size_t size, struct grub_relocator_chunk *res,
int from_low_priv, int collisioncheck)
{
grub_mm_region_t rb, rbp;
grub_mm_header_t hb = NULL, hbp = NULL;
grub_addr_t best_addr;
again:
rb = NULL, rbp = NULL;
{
grub_mm_region_t r, rp;
for (rp = NULL, r = grub_mm_base; r; rp = r, r = r->next)
{
grub_dprintf ("relocator", "region %p. %d %d %d %d\n", r,
(grub_addr_t) r + r->size + sizeof (*r) >= start,
(grub_addr_t) r < end, r->size + sizeof (*r) >= size,
(rb == NULL || (from_low_priv ? rb > r : rb < r)));
if ((grub_addr_t) r + r->size + sizeof (*r) >= start
&& (grub_addr_t) r < end && r->size + sizeof (*r) >= size
&& (rb == NULL || (from_low_priv ? rb > r : rb < r)))
{
rb = r;
rbp = rp;
}
}
}
if (!rb)
{
grub_dprintf ("relocator", "no suitable region found\n");
return 0;
}
grub_dprintf ("relocator", "trying region %p - %p\n", rb, rb + rb->size + 1);
hb = get_best_header (rel, start, end, align, size, rb, &hbp, &best_addr,
from_low_priv, collisioncheck);
grub_dprintf ("relocator", "best header %p/%p/%lx\n", hb, hbp,
(unsigned long) best_addr);
if (!hb)
{
if (from_low_priv)
start = (grub_addr_t) (rb + rb->size + sizeof (*rb));
else
end = (grub_addr_t) rb - 1;
goto again;
}
/* Special case: relocating region start. */
if (best_addr < (grub_addr_t) hb)
{
grub_addr_t newreg_start, newreg_raw_start = best_addr + size;
grub_addr_t newreg_start, newreg_raw_start = addr + size;
grub_addr_t newreg_size, newreg_presize;
grub_mm_header_t new_header;
grub_mm_header_t hb = (grub_mm_header_t) (rb + 1);
res->src = best_addr;
res->type = CHUNK_TYPE_REGION_START;
res->host_start = (grub_addr_t) rb - rb->pre_size;
grub_dprintf ("relocator", "ra = %p, rb = %p\n", regancestor, rb);
newreg_start = ALIGN_UP (newreg_raw_start, GRUB_MM_ALIGN);
newreg_presize = newreg_start - newreg_raw_start;
@ -295,17 +144,14 @@ malloc_in_range (struct grub_relocator *rel,
struct grub_mm_header *newregfirst = rb->first;
struct grub_mm_region *newregnext = rb->next;
struct grub_mm_region *newreg = (void *) newreg_start;
hbp->next = new_header;
hancestor->next = new_header;
if (newregfirst == hb)
newregfirst = new_header;
newreg->first = newregfirst;
newreg->next = newregnext;
newreg->pre_size = newreg_presize;
newreg->size = newreg_size;
if (rbp)
rbp->next = newreg;
else
grub_mm_base = newreg;
*regancestor = newreg;
{
grub_mm_header_t h = newreg->first, hp = NULL;
do
@ -313,31 +159,38 @@ malloc_in_range (struct grub_relocator *rel,
if ((void *) h < (void *) (newreg + 1))
grub_fatal ("Failed to adjust memory region: %p, %p, %p, %p, %p",
newreg, newreg->first, h, hp, hb);
if ((void *) h == (void *) (newreg + 1))
grub_dprintf ("relocator",
"Free start memory region: %p, %p, %p, %p, %p",
newreg, newreg->first, h, hp, hb);
hp = h;
h = h->next;
}
while (h != newreg->first);
}
}
return 1;
}
static void
allocate_inreg (grub_addr_t addr, grub_size_t size,
grub_mm_header_t hb, grub_mm_header_t hbp,
grub_mm_region_t rb)
{
struct grub_mm_header *foll = NULL;
res->src = best_addr;
res->type = CHUNK_TYPE_IN_REGION;
if (ALIGN_UP (best_addr + size, GRUB_MM_ALIGN) + GRUB_MM_ALIGN
if (ALIGN_UP (addr + size, GRUB_MM_ALIGN) + GRUB_MM_ALIGN
<= (grub_addr_t) (hb + hb->size))
{
foll = (void *) ALIGN_UP (best_addr + size, GRUB_MM_ALIGN);
foll = (void *) ALIGN_UP (addr + size, GRUB_MM_ALIGN);
foll->magic = GRUB_MM_FREE_MAGIC;
foll->size = hb->size - (foll - hb);
}
if (best_addr - (grub_addr_t) hb >= sizeof (*hb))
if (addr - (grub_addr_t) hb >= sizeof (*hb))
{
hb->size = ((best_addr - (grub_addr_t) hb) >> GRUB_MM_ALIGN_LOG2);
hb->size = ((addr - (grub_addr_t) hb) >> GRUB_MM_ALIGN_LOG2);
if (foll)
{
foll->next = hb;
@ -359,8 +212,363 @@ malloc_in_range (struct grub_relocator *rel,
if (rb->first == hb)
rb->first = (void *) (rb + 1);
}
return 1;
}
static int
malloc_in_range (struct grub_relocator *rel,
grub_addr_t start, grub_addr_t end, grub_addr_t align,
grub_size_t size, struct grub_relocator_chunk *res,
int from_low_priv, int collisioncheck)
{
grub_mm_region_t r, *ra, base_saved;
struct event *events = NULL, *eventt = NULL, *t;
unsigned maxevents = 2;
grub_mm_header_t p, pa;
unsigned *counter;
int nallocs = 0;
unsigned i, j, N = 0;
grub_addr_t target = 0;
grub_dprintf ("relocator",
"trying to allocate in %x-%x aligned %x size %x\n",
start, end, align, size);
start = ALIGN_UP (start, align);
end = ALIGN_DOWN (end - size, align) + size;
grub_dprintf ("relocator",
"trying to allocate in %x-%x aligned %x size %x\n",
start, end, align, size);
if (end < start + size)
return 0;
/* We have to avoid any allocations when filling scanline events.
Hence 2-stages.
*/
for (r = grub_mm_base; r; r = r->next)
{
p = r->first;
do
{
maxevents += 2;
p = p->next;
}
while (p != r->first);
maxevents += 2;
}
if (collisioncheck && rel)
{
struct grub_relocator_chunk *chunk;
for (chunk = rel->chunks; chunk; chunk = chunk->next)
maxevents += 2;
}
events = grub_malloc (maxevents * sizeof (events[0]));
eventt = grub_malloc (maxevents * sizeof (events[0]));
counter = grub_malloc ((DIGITSORT_MASK + 2) * sizeof (counter[0]));
if (!events || !eventt || !counter)
{
grub_dprintf ("relocator", "events or counter allocation failed %d\n",
maxevents);
grub_free (events);
grub_free (eventt);
grub_free (counter);
return 0;
}
if (collisioncheck && rel)
{
struct grub_relocator_chunk *chunk;
for (chunk = rel->chunks; chunk; chunk = chunk->next)
{
events[N].type = COLLISION_START;
events[N].pos = chunk->target;
N++;
events[N].type = COLLISION_END;
events[N].pos = chunk->target + chunk->size;
N++;
}
}
/* No malloc from this point. */
base_saved = grub_mm_base;
grub_mm_base = NULL;
for (ra = &base_saved, r = *ra; r; ra = &(r->next), r = *ra)
{
int pre_added = 0;
pa = r->first;
p = pa->next;
do
{
if (p == (grub_mm_header_t) (r + 1))
{
pre_added = 1;
events[N].type = REG_BEG_START;
events[N].pos = (grub_addr_t) r - r->pre_size;
events[N].reg = r;
events[N].regancestor = ra;
events[N].head = p;
events[N].hancestor = pa;
N++;
events[N].type = REG_BEG_END;
events[N].pos = (grub_addr_t) (p + p->size) - sizeof (*r);
N++;
}
else
{
events[N].type = IN_REG_START;
events[N].pos = (grub_addr_t) p;
events[N].head = p;
events[N].hancestor = pa;
events[N].reg = r;
N++;
events[N].type = IN_REG_END;
events[N].pos = (grub_addr_t) (p + p->size);
N++;
}
pa = p;
p = pa->next;
}
while (pa != r->first);
/* FIXME */
if (0)//if (!pre_added)
{
events[N].type = REG_BEG_START;
events[N].pos = (grub_addr_t) r - r->pre_size;
events[N].reg = r;
N++;
events[N].type = REG_BEG_END;
events[N].pos = (grub_addr_t) r;
N++;
}
}
/* Put ending events after starting events. */
{
int st = 0, e = N / 2;
for (j = 0; j < N; j++)
if (is_start (events[j].type) || events[j].type == COLLISION_START)
eventt[st++] = events[j];
else
eventt[e++] = events[j];
t = eventt;
eventt = events;
events = t;
}
for (i = 0; i < (BITS_IN_BYTE * sizeof (grub_addr_t) / DIGITSORT_BITS);
i++)
{
memset (counter, 0, (1 + (1 << DIGITSORT_BITS)) * sizeof (counter[0]));
for (j = 0; j < N; j++)
counter[((events[j].pos >> (DIGITSORT_BITS * i))
& DIGITSORT_MASK) + 1]++;
for (j = 0; j <= DIGITSORT_MASK; j++)
counter[j+1] += counter[j];
for (j = 0; j < N; j++)
eventt[counter[((events[j].pos >> (DIGITSORT_BITS * i))
& DIGITSORT_MASK)]++] = events[j];
t = eventt;
eventt = events;
events = t;
}
grub_dprintf ("relocator", "scanline events:\n");
for (j = 0; j < N; j++)
grub_dprintf ("relocator", "event %x, type %d\n", events[j].pos,
events[j].type);
/* Now events are nicely sorted. */
if (from_low_priv)
{
int nstarted = 0, ncollisions = 0;
grub_addr_t starta = 0;
int numstarted;
for (j = 0; j < N; j++)
{
switch (events[j].type)
{
case COLLISION_END:
ncollisions--;
case IN_REG_START:
case REG_BEG_START:
if ((events[j].type == COLLISION_END ? nstarted != 0
: nstarted == 0)
&& ncollisions == 0)
{
starta = ALIGN_UP (events[j].pos, align);
numstarted = j;
}
if (events[j].type != COLLISION_END)
nstarted++;
break;
case IN_REG_END:
case REG_BEG_END:
nstarted--;
case COLLISION_START:
if (((events[j].type == COLLISION_START)
? nstarted != 0 : nstarted == 0)
&& ncollisions == 0)
{
target = starta;
if (target < start)
target = start;
grub_dprintf ("relocator", "%x, %x, %x\n", target, start,
events[j].pos);
if (target + size <= end && target + size <= events[j].pos)
/* Found an usable address. */
goto found;
}
if (events[j].type == COLLISION_START)
ncollisions++;
break;
}
}
}
else
{
int nstarted = 0, ncollisions = 0;
grub_addr_t enda = 0;
int numend;
for (j = N - 1; j != (unsigned) -1; j--)
{
switch (events[j].type)
{
case COLLISION_START:
ncollisions--;
case IN_REG_END:
case REG_BEG_END:
if ((events[j].type == COLLISION_END ? nstarted != 0
: nstarted == 0)
&& ncollisions == 0)
{
enda = ALIGN_DOWN (events[j].pos - size, align) + size;
numend = j;
}
nstarted++;
break;
case IN_REG_START:
case REG_BEG_START:
nstarted--;
case COLLISION_END:
if ((events[j].type == COLLISION_START ? nstarted != 0
: nstarted == 0)
&& ncollisions == 0)
{
target = enda - size;
if (target > end - size)
target = end - size;
grub_dprintf ("relocator", "%x, %x, %x\n", target, start,
events[j].pos);
if (target >= start && target >= events[j].pos)
goto found;
}
if (events[j].type == COLLISION_START)
ncollisions++;
break;
}
}
}
grub_mm_base = base_saved;
grub_free (events);
grub_free (eventt);
grub_free (counter);
return 0;
found:
{
int last_start = 0;
for (j = 0; j < N; j++)
{
if (j != 0 && events[j - 1].pos != events[j].pos)
{
grub_addr_t alloc_start, alloc_end;
alloc_start = max (events[j - 1].pos, target);
alloc_end = min (events[j].pos, target + size);
if (alloc_end > alloc_start)
{
grub_dprintf ("relocator", "%d\n", last_start);
if (events[last_start].type == REG_BEG_START
|| events[last_start].type == IN_REG_START)
{
if (events[last_start].type == REG_BEG_START &&
(grub_addr_t) (events[last_start].reg + 1) > target)
allocate_regstart (alloc_start, alloc_end - alloc_start,
events[last_start].reg,
events[last_start].regancestor,
events[last_start].hancestor);
else
allocate_inreg (alloc_start, alloc_end - alloc_start,
events[last_start].head,
events[last_start].hancestor,
events[last_start].reg);
}
nallocs++;
}
}
if (is_start (events[j].type))
last_start = j;
}
}
grub_memset ((void *) target, 0, size);
grub_dprintf ("relocator", "baseptr = %p\n", &base_saved);
for (r = base_saved; r; r = r->next)
{
p = r->first;
do
{
if (!p)
grub_fatal ("null in the ring %p %p\n", r, p);
p = p->next;
}
while (p != r->first);
}
/* Malloc is available again. */
grub_mm_base = base_saved;
/* FIXME: react on out of memory. */
res->subchunks = grub_malloc (sizeof (res->subchunks[0]) * nallocs);
res->nsubchunks = nallocs;
{
int last_start = 0;
int cural = 0;
for (j = 0; j < N; j++)
{
if (j != 0 && events[j - 1].pos != events[j].pos)
{
grub_addr_t alloc_start, alloc_end;
alloc_start = max (events[j - 1].pos, target);
alloc_end = min (events[j].pos, target + size);
if (alloc_end > alloc_start)
{
res->subchunks[cural].start = alloc_start;
res->subchunks[cural].size = alloc_end - alloc_start;
if (res->subchunks[last_start].type == IN_REG_START)
res->subchunks[cural].type = CHUNK_TYPE_IN_REGION;
else if (res->subchunks[last_start].type == REG_BEG_START)
{
res->subchunks[cural].type = CHUNK_TYPE_REGION_START;
res->subchunks[cural].host_start
= (grub_addr_t) events[last_start].reg;
}
cural++;
}
}
if (is_start (events[j].type))
last_start = j;
}
}
res->src = target;
res->size = size;
grub_dprintf ("relocator", "allocated: %x %x\n", target, size);
return 1;
}
static void
@ -601,22 +809,28 @@ grub_relocator_unload (struct grub_relocator *rel)
return;
for (chunk = rel->chunks; chunk; chunk = next)
{
switch (chunk->type)
unsigned i;
for (i = 0; i < chunk->nsubchunks; i++)
switch (chunk->subchunks[i].type)
{
case CHUNK_TYPE_REGION_START:
{
grub_mm_region_t r1, r2, *rp;
grub_mm_header_t h;
grub_size_t pre_size;
r1 = (grub_mm_region_t) ALIGN_UP (chunk->src + chunk->size,
r1 = (grub_mm_region_t) ALIGN_UP (chunk->subchunks[i].start
+ chunk->subchunks[i].size,
GRUB_MM_ALIGN);
r2 = (grub_mm_region_t) ALIGN_UP (chunk->subchunks[i].host_start,
GRUB_MM_ALIGN);
r2 = (grub_mm_region_t) ALIGN_UP (chunk->host_start, GRUB_MM_ALIGN);
for (rp = &grub_mm_base; *rp && *rp != r2; rp = &((*rp)->next));
/* FIXME */
if (!*rp)
grub_fatal ("Anomaly in region alocations detected. "
"Simultaneous relocators?");
pre_size = ALIGN_UP (chunk->host_start, GRUB_MM_ALIGN)
- chunk->host_start;
pre_size = ALIGN_UP (chunk->subchunks[i].host_start,
GRUB_MM_ALIGN)
- chunk->subchunks[i].host_start;
r2->first = r1->first;
r2->next = r1->next;
r2->pre_size = pre_size;
@ -631,11 +845,12 @@ grub_relocator_unload (struct grub_relocator *rel)
}
case CHUNK_TYPE_IN_REGION:
{
grub_mm_header_t h = (grub_mm_header_t) ALIGN_DOWN (chunk->src,
grub_mm_header_t h
= (grub_mm_header_t) ALIGN_DOWN (chunk->subchunks[i].start,
GRUB_MM_ALIGN);
h->size = (chunk->src / GRUB_MM_ALIGN)
- ((chunk->src + chunk->size + GRUB_MM_ALIGN - 1)
/ GRUB_MM_ALIGN);
h->size = (chunk->subchunks[i].start / GRUB_MM_ALIGN)
- ((chunk->subchunks[i].start + chunk->subchunks[i].size
+ GRUB_MM_ALIGN - 1) / GRUB_MM_ALIGN);
h->next = h;
h->magic = GRUB_MM_ALLOC_MAGIC;
grub_free (h + 1);
@ -643,6 +858,7 @@ grub_relocator_unload (struct grub_relocator *rel)
}
}
next = chunk->next;
grub_free (chunk->subchunks);
grub_free (chunk);
}
}