Bug 99167

Summary: aio on hugepages BUGs
Product: Red Hat Enterprise Linux 3 Reporter: Jason Baron <jbaron>
Component: kernelAssignee: Ingo Molnar <mingo>
Status: CLOSED RAWHIDE QA Contact: Brian Brock <bbrock>
Severity: medium Docs Contact:
Priority: medium    
Version: 3.0CC: jbaron, knoel, riel, sct
Target Milestone: ---   
Target Release: ---   
Hardware: All   
OS: Linux   
Whiteboard:
Fixed In Version: Doc Type: Bug Fix
Doc Text:
Story Points: ---
Clone Of: Environment:
Last Closed: 2003-07-18 13:29:51 UTC Type: ---
Regression: --- Mount Type: ---
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: --- RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: --- Target Upstream Version:
Embargoed:
Bug Depends On:    
Bug Blocks: 97140    

Description Jason Baron 2003-07-15 13:39:43 UTC
From Bugzilla Helper:
User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.0.1) Gecko/20020830

Description of problem:
seems like page counts for AIO arent' quite right. i've attached the 
reproducer. The program simply attempts to aio read a file into a hugepage 512
bytes at a time.


Jul 14 13:27:27 beast1 kernel:  hm, tmp: 039d802c (0), ->mapping: 2d2190c8
Jul 14 13:27:27 beast1 kernel: invalid operand: 0000
Jul 14 13:27:27 beast1 kernel: parport_pc lp parport autofs pcnet32 mii 
crc32 floppy microcode st keybdev mousedev hid input usb-ohci usbcore ext3 
jbd ips aic7xxx sd_mod scsi_mod
Jul 14 13:27:27 beast1 kernel: CPU:    1
Jul 14 13:27:27 beast1 kernel: EIP:    0060:[<02120aca>]    Not tainted
Jul 14 13:27:27 beast1 kernel: EFLAGS: 00010206
Jul 14 13:27:27 beast1 kernel:
Jul 14 13:27:27 beast1 kernel: EIP is at check_huge_page [kernel] 0x8a 
(2.4.21-1.1931.2.320.entsmp)
Jul 14 13:27:27 beast1 kernel: eax: 0000002b   ebx: 039d802c   ecx: 
00000000   edx: 02374e14
Jul 14 13:27:27 beast1 kernel: esi: 00000000   edi: 2ff0ad28   ebp: 
00000001   esp: 2b7ebda0
Jul 14 13:27:27 beast1 kernel: ds: 0068   es: 0068   ss: 0068
Jul 14 13:27:27 beast1 kernel: Process aio (pid: 2972, stackpage=2b7eb000)
Jul 14 13:27:27 beast1 kernel: Stack: 022ae3e0 039d802c 00000000 2d2190c8 
039d802c 2ff0ad14 021210d1 039d802c
Jul 14 13:27:27 beast1 kernel:        2ff0ad1c 021547c9 039d802c 0213ffa5 
2ff0ad14 2bbd3230 00000200 00000000
Jul 14 13:27:27 beast1 kernel:        02187641 00aaadd4 00000200 2f83b18c 
00000000 00000000 021876f7 00000000
Jul 14 13:27:27 beast1 kernel: Call Trace:   [<021210d1>] free_huge_page 
[kernel] 0x21 (0x2b7ebdb8)
Jul 14 13:27:27 beast1 kernel: [<021547c9>] __free_pages [kernel] 0x39 
(0x2b7ebdc4)
Jul 14 13:27:27 beast1 kernel: [<0213ffa5>] unmap_kvec [kernel] 0x45 
(0x2b7ebdcc)
Jul 14 13:27:27 beast1 kernel: [<02187641>] generic_aio_complete_rw 
[kernel] 0x31 (0x2b7ebde0)
Jul 14 13:27:27 beast1 kernel: [<021876f7>] generic_aio_complete_read 
[kernel] 0x27 (0x2b7ebdf8)
Jul 14 13:27:27 beast1 kernel: [<02147ec7>] __iodesc_free [kernel] 0x97 
(0x2b7ebe0c)
Jul 14 13:27:27 beast1 kernel: [<021876d0>] generic_aio_complete_read 
[kernel] 0x0 (0x2b7ebe20)
Jul 14 13:27:27 beast1 kernel: [<021487ab>] __iodesc_read_finish [kernel] 
0x22b (0x2b7ebe38)
Jul 14 13:27:27 beast1 kernel: [<02148fd0>] rw_kvec_core [kernel] 0x1d0 
(0x2b7ebe74)
Jul 14 13:27:27 beast1 kernel: [<02148bf7>] generic_file_kvec_read 
[kernel] 0x47 (0x2b7ebeb4)
Jul 14 13:27:27 beast1 kernel: [<021876d0>] generic_aio_complete_read 
[kernel] 0x0 (0x2b7ebec4)
Jul 14 13:27:27 beast1 kernel: [<021877d8>] generic_aio_rw [kernel] 0xa8 
(0x2b7ebed8)
Jul 14 13:27:27 beast1 kernel: [<021876d0>] generic_aio_complete_read 
[kernel] 0x0 (0x2b7ebee4)
Jul 14 13:27:27 beast1 kernel: [<021876d0>] generic_aio_complete_read 
[kernel] 0x0 (0x2b7ebefc)
Jul 14 13:27:27 beast1 kernel: [<0218787e>] generic_file_aio_read [kernel] 
0x2e (0x2b7ebf18)
Jul 14 13:27:27 beast1 kernel: [<02187296>] io_submit_one [kernel] 0x186 
(0x2b7ebf30)
Jul 14 13:27:27 beast1 kernel: [<0218747f>] sys_io_submit [kernel] 0xcf 



Version-Release number of selected component (if applicable):


How reproducible:
Always

Steps to Reproduce:
1.run attached program
2.
3.
    

Actual Results:  BUG check

Expected Results:  no BUG

Additional info:

Comment 1 Jason Baron 2003-07-15 13:41:11 UTC
here is a program to cause the BUG check:

#include <stdio.h>
#include <errno.h>
#include <assert.h>
#include <stdlib.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/shm.h>

#include <libaio.h>



io_context_t    io_ctx;
#define BAD_CTX ((io_context_t)-1)

void aio_setup(int n)
{
        int res = io_queue_init(n, &io_ctx);
        if (res != 0) {
                printf("io_queue_setup(%d) returned %d (%s)\n",
                        n, res, strerror(-res));
                exit(3);
        }
}

int attempt_io_submit(io_context_t ctx, long nr, struct iocb *ios[], int expect)
{
        int res;

        printf("expect %3d: io_submit(%10p, %3ld, %10p) = ", expect, ctx, nr, ios);
        fflush(stdout);
        res = io_submit(ctx, nr, ios);
        printf("%3d [%s]%s\n", res, (res <= 0) ? strerror(-res) : "",
                (res != expect) ? " -- FAILED" : "");
        if (res != expect)
                return 1;

        return 0;
}

int sync_submit(struct iocb *iocb)
{
        struct io_event event;
        struct iocb *iocbs[] = { iocb };
        int res;

        /* 30 second timeout should be enough */
        struct timespec ts;
        ts.tv_sec = 30;
        ts.tv_nsec = 0;

        res = io_submit(io_ctx, 1, iocbs);
        if (res != 1) {
                printf("sync_submit: io_submit res=%d [%s]\n", res, strerror(-res));
                return res;
        }

        res = io_getevents(io_ctx, 0, 1, &event, &ts);
        if (res != 1) {
                printf("sync_submit: io_getevents res=%d [%s]\n", res,
strerror(-res));
                return res;
        }
        return event.res;
}

#define SETUP   aio_setup(1024)


#define READ            'r'
#define WRITE           'w'
#define READ_SILENT     'R'
#define WRITE_SILENT    'W'
int attempt_rw(int fd, void *buf, int count, long long pos, int rw, int expect)
{
        struct iocb iocb;
        int res;
        int silent = 0;

        switch(rw) {
        case READ_SILENT:
                silent = 1;
        case READ:
                io_prep_pread (&iocb, fd, buf, count, pos);
                break;
        case WRITE_SILENT:
                silent = 1;
        case WRITE:
                io_prep_pwrite(&iocb, fd, buf, count, pos);
                break;
        }

        if (!silent) {
                printf("expect %5d: (%c), res = ", expect, rw);
                fflush(stdout);
        }
        res = sync_submit(&iocb);
        if (!silent || res != expect) {
                if (silent)
                        printf("expect %5d: (%c), res = ", expect, rw);
                printf("%5d [%s]%s\n", res,
                        (res <= 0) ? strerror(-res) : "Success",
                        (res != expect) ? " -- FAILED" : "");
        }

        if (res != expect)
                return 1;

        return 0;
}

#define LPAGE_SIZE      (2UL*1024UL*1024UL)


int main(void)
{
#define SIZE    512
        char buf2[SIZE];
        char *buf;
        int rwfd;
        int     status = 0;
        int shmid;
        int res, i;

        SETUP;
        rwfd = open("/tmp/foo", O_CREAT | O_RDWR);              assert(rwfd != -1);
        res = ftruncate(rwfd, 512);                     assert(res == 0);

        if((shmid = shmget(4, LPAGE_SIZE, SHM_HUGETLB|IPC_CREAT|SHM_R|SHM_W)) < 0) {
                perror("Failure:");
                exit(1);
        }
        printf("shmid: 0x%x\n", shmid);
        buf = shmat(shmid, NULL, SHM_RND) ;
        printf("attached at: %p\n", buf);
        if (errno != 0) {
                perror("Shared Memory Attach Failure:");
                exit(2);
        }

        for(i=0;i<LPAGE_SIZE/SIZE;i++) {
                status |= attempt_rw(rwfd, &buf[i*SIZE], SIZE, 0, READ, SIZE);
        }

        close(rwfd);

        return status;

}




Comment 2 Ingo Molnar 2003-07-17 10:26:00 UTC
the cause of the bug was a hugetlb-unaware line in mm_map_user_kvec(), sent the
fix. It fixes your testcase. Please retest with the next build.

Comment 3 Jay Turner 2003-07-18 13:29:51 UTC
For anyone else that happens to come across this bug, need to actually allocate
hugepages before attempting to use them:

echo "20" > proc/sys/vm/hugetlb_pool

Anyway, the fix is confirmed with 2.4.21-1.1931.2.349.2.2.ent